diff --git a/ydb/core/viewer/json_storage.h b/ydb/core/viewer/json_storage.h index 303e7d2ce4ae..d763576bd1b0 100644 --- a/ydb/core/viewer/json_storage.h +++ b/ydb/core/viewer/json_storage.h @@ -125,7 +125,6 @@ class TJsonStorage : public TJsonStorageBase { } void Bootstrap() override { - Cerr << "iiiii Bootstrap" << Endl; TIntrusivePtr domains = AppData()->DomainsInfo; ui64 hiveId = domains->GetHive(); if (hiveId != TDomainsInfo::BadTabletId) { @@ -251,7 +250,7 @@ class TJsonStorage : public TJsonStorageBase { } bool CheckGroupFilters(const TString& groupId, const TString& poolName, const TGroupRow& groupRow) { - if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) { + if (!EffectiveGroupFilter.contains(groupId)) { return false; } switch (With) { @@ -498,6 +497,7 @@ struct TJsonRequestParameters { {"name":"tenant","in":"query","description":"tenant name","required":false,"type":"string"}, {"name":"pool","in":"query","description":"storage pool name","required":false,"type":"string"}, {"name":"node_id","in":"query","description":"node id","required":false,"type":"integer"}, + {"name":"pdisk_id","in":"query","description":"pdisk id","required":false,"type":"integer"}, {"name":"group_id","in":"query","description":"group id","required":false,"type":"integer"}, {"name":"need_groups","in":"query","description":"return groups information","required":false,"type":"boolean","default":true}, {"name":"need_disks","in":"query","description":"return disks information","required":false,"type":"boolean","default":true}, diff --git a/ydb/core/viewer/json_storage_base.h b/ydb/core/viewer/json_storage_base.h index d9763a16ea10..1924dcdb2d89 100644 --- a/ydb/core/viewer/json_storage_base.h +++ b/ydb/core/viewer/json_storage_base.h @@ -72,10 +72,11 @@ class TJsonStorageBase : public TViewerPipeClient { ui32 Timeout = 0; TString FilterTenant; THashSet FilterStoragePools; - TVector FilterGroupIds; TString Filter; + std::unordered_set FilterGroupIds; std::unordered_set FilterNodeIds; - THashSet EffectiveFilterGroupIds; + std::unordered_set FilterPDiskIds; + THashSet EffectiveGroupFilter; std::unordered_set NodeIds; bool NeedAdditionalNodesRequests; @@ -133,9 +134,9 @@ class TJsonStorageBase : public TViewerPipeClient { FilterStoragePools.emplace(filterStoragePool); } SplitIds(params.Get("node_id"), ',', FilterNodeIds); + SplitIds(params.Get("pdisk_id"), ',', FilterPDiskIds); NeedAdditionalNodesRequests = !FilterNodeIds.empty(); SplitIds(params.Get("group_id"), ',', FilterGroupIds); - Sort(FilterGroupIds); Filter = params.Get("filter"); if (params.Get("with") == "missing") { With = EWith::MissingDisks; @@ -263,7 +264,6 @@ class TJsonStorageBase : public TViewerPipeClient { } void Handle(TEvInterconnect::TEvNodesInfo::TPtr& ev) { - Cerr << "iiiiiiiiii TEvNodesInfo 1" << Endl; ui32 maxAllowedNodeId = std::numeric_limits::max(); TIntrusivePtr dynamicNameserviceConfig = AppData()->DynamicNameserviceConfig; if (dynamicNameserviceConfig) { @@ -275,7 +275,6 @@ class TJsonStorageBase : public TViewerPipeClient { SendNodeRequests(ni.NodeId); } } - Cerr << "iiiiiiiiii TEvNodesInfo 2" << Endl; RequestDone(); } @@ -314,7 +313,6 @@ class TJsonStorageBase : public TViewerPipeClient { } void Handle(TEvHive::TEvResponseHiveStorageStats::TPtr& ev) { - Cerr << "iiiii TEvResponseHiveStorageStats" << Endl; HiveStorageStats[ev->Cookie] = ev->Release(); RequestDone(); } @@ -354,41 +352,39 @@ class TJsonStorageBase : public TViewerPipeClient { } void Handle(TEvWhiteboard::TEvVDiskStateResponse::TPtr& ev) { - Cerr << "iiiii TEvVDiskStateResponse" << Endl; ui64 nodeId = ev.Get()->Cookie; - Cerr << "iiiii nodeid" << nodeId << Endl; auto& vDiskInfo = VDiskInfo[nodeId] = std::move(ev->Get()->Record); for (auto& vDiskStateInfo : *(vDiskInfo.MutableVDiskStateInfo())) { vDiskStateInfo.SetNodeId(nodeId); VDiskId2vDiskStateInfo[VDiskIDFromVDiskID(vDiskStateInfo.GetVDiskId())] = &vDiskStateInfo; + + if ((FilterNodeIds.empty() || FilterNodeIds.contains(nodeId)) + && (FilterPDiskIds.empty() || FilterPDiskIds.contains(vDiskStateInfo.GetPDiskId())) + && (FilterGroupIds.empty() || FilterGroupIds.contains(ToString(vDiskStateInfo.GetVDiskId().GetGroupID())))) { + EffectiveGroupFilter.insert(ToString(vDiskStateInfo.GetVDiskId().GetGroupID())); + } } RequestDone(); } void Handle(TEvWhiteboard::TEvPDiskStateResponse::TPtr& ev) { ui64 nodeId = ev.Get()->Cookie; + PDiskInfo[nodeId] = std::move(ev->Get()->Record); RequestDone(); } void Handle(TEvWhiteboard::TEvBSGroupStateResponse::TPtr& ev) { ui64 nodeId = ev.Get()->Cookie; - Cerr << "iiiiiiiiiiiii Cookie " << nodeId << Endl; for (const auto& info : ev->Get()->Record.GetBSGroupStateInfo()) { TString storagePoolName = info.GetStoragePoolName(); if (storagePoolName.empty()) { continue; } - Cerr << "iiiiiiiiiiiii nodeId " << info.GetNodeId() << Endl; if (FilterNodeIds.empty() || FilterNodeIds.contains(info.GetNodeId())) { StoragePoolInfo[storagePoolName].Groups.emplace(ToString(info.GetGroupID())); - TString groupId(ToString(info.GetGroupID())); - if (FilterGroupIds.empty() || BinarySearch(FilterGroupIds.begin(), FilterGroupIds.end(), groupId)) { - EffectiveFilterGroupIds.insert(groupId); - } } for (const auto& vDiskNodeId : info.GetVDiskNodeIds()) { - Cerr << "iiiiiiiiiiiii vDiskNodeId " << vDiskNodeId << Endl; Group2NodeId[info.GetGroupID()].push_back(vDiskNodeId); } } diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index 53b615d2b0b9..c0278feb6122 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -269,7 +269,6 @@ Y_UNIT_TEST_SUITE(Viewer) { auto sample = nodes[0]; nodes.clear(); - for (int nodeId = 0; nodeId < nodesTotal; nodeId++) { sample.NodeId = nodeId; nodes.emplace_back(sample); @@ -1107,17 +1106,36 @@ Y_UNIT_TEST_SUITE(Viewer) { } void ChangeBSGroupStateResponse(TEvWhiteboard::TEvBSGroupStateResponse::TPtr* ev) { + ui64 nodeId = (*ev)->Cookie; auto& pbRecord = (*ev)->Get()->Record; - // pbRecord.clear_bsgroupstateinfo(); - // auto state = pbRecord.add_bsgroupstateinfo(); - Cerr << pbRecord.ShortDebugString() << Endl; + + pbRecord.clear_bsgroupstateinfo(); + + for (ui64 groupId = 1; groupId <= 9; groupId++) { + if (groupId == nodeId) { + continue; + } + auto state = pbRecord.add_bsgroupstateinfo(); + state->set_groupid(groupId); + state->set_storagepoolname("/Root:test"); + state->set_nodeid(nodeId); + for (int k = 1; k <= 8; k++) { + auto vdisk = groupId * 8 + k; + auto vdiskId = state->add_vdiskids(); + vdiskId->set_groupid(groupId); + vdiskId->set_groupgeneration(1); + vdiskId->set_vdisk(vdisk); + } + } } void ChangePDiskStateResponse(TEvWhiteboard::TEvPDiskStateResponse::TPtr* ev) { auto& pbRecord = (*ev)->Get()->Record; - // pbRecord.clear_pdiskstateinfo(); - // auto state = pbRecord.add_pdiskstateinfo(); - Cerr << pbRecord.ShortDebugString() << Endl; + pbRecord.clear_pdiskstateinfo(); + for (int k = 0; k < 2; k++) { + auto state = pbRecord.add_pdiskstateinfo(); + state->set_pdiskid(k); + } } @@ -1125,32 +1143,46 @@ Y_UNIT_TEST_SUITE(Viewer) { ui64 nodeId = (*ev)->Cookie; auto& pbRecord = (*ev)->Get()->Record; - auto sample = pbRecord.vdiskstateinfo(0); pbRecord.clear_vdiskstateinfo(); - for (int k = 1; k <= 8; k++) { - auto groupId = (nodeId + k) % 9; - ui32 pdisk = k <= 4 ? 0 : 1; + for (int k = 0; k < 8; k++) { + auto groupId = (nodeId + k) % 9 + 1; + auto vdisk = groupId * 8 + k + 1; + ui32 pdisk = k / 4; ui32 slotid = k % 4; - auto vdisk = 0; auto state = pbRecord.add_vdiskstateinfo(); - state->CopyFrom(sample); state->set_pdiskid(pdisk); state->set_vdiskslotid(slotid); - state->mutable_vdiskid()->set_vdisk(vdisk++); state->mutable_vdiskid()->set_groupid(groupId); + state->mutable_vdiskid()->set_groupgeneration(1); + state->mutable_vdiskid()->set_vdisk(vdisk++); state->set_vdiskstate(NKikimrWhiteboard::EVDiskState::OK); state->set_nodeid(nodeId); } } - void JsonStorage9NodesListingTest() { + void AddGroupsInControllerSelectGroupsResult(TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr* ev, int groupCount) { + auto& pbRecord = (*ev)->Get()->Record; + auto pbMatchGroups = pbRecord.mutable_matchinggroups(0); + + auto sample = pbMatchGroups->groups(0); + pbMatchGroups->ClearGroups(); + + for (int groupId = 1; groupId <= groupCount; groupId++) { + auto group = pbMatchGroups->add_groups(); + group->CopyFrom(sample); + group->set_groupid(groupId++); + group->set_storagepoolname("/Root:test"); + } + }; + + void JsonStorage9Nodes9GroupsListingTest(TString version, bool groupFilter, bool nodeFilter, bool pdiskFilter, ui32 expectedFoundGroups, ui32 expectedTotalGroups) { TPortManager tp; ui16 port = tp.GetPort(2134); ui16 grpcPort = tp.GetPort(2135); auto settings = TServerSettings(port); settings.InitKikimrRunConfig() - .SetNodeCount(1) + .SetNodeCount(9) .SetUseRealThreads(false) .SetDomainName("Root"); TServer server(settings); @@ -1163,8 +1195,16 @@ Y_UNIT_TEST_SUITE(Viewer) { THttpRequest httpReq(HTTP_METHOD_GET); httpReq.CgiParameters.emplace("with", "all"); - httpReq.CgiParameters.emplace("version", "v2"); - // httpReq.CgiParameters.emplace("node_id", "1"); + httpReq.CgiParameters.emplace("version", version); + if (groupFilter) { + httpReq.CgiParameters.emplace("group_id", "1"); + } + if (nodeFilter) { + httpReq.CgiParameters.emplace("node_id", "1"); + } + if (pdiskFilter) { + httpReq.CgiParameters.emplace("pdisk_id", "0"); + } auto page = MakeHolder("viewer", "title"); TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/storage", nullptr); auto request = MakeHolder(monReq); @@ -1172,32 +1212,32 @@ Y_UNIT_TEST_SUITE(Viewer) { auto observerFunc = [&](TAutoPtr& ev) { Y_UNUSED(ev); switch (ev->GetTypeRewrite()) { - case TEvInterconnect::EvNodesInfo: { - auto *x = reinterpret_cast(&ev); - Cerr << "aaaa EvNodesInfo 1" << Endl; - ChangeListNodes(x, 9); - Cerr << "aaaa EvNodesInfo 2" << Endl; + case NConsole::TEvConsole::EvListTenantsResponse: { + auto *x = reinterpret_cast(&ev); + Ydb::Cms::ListDatabasesResult listTenantsResult; + (*x)->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult); + listTenantsResult.Addpaths("/Root"); + (*x)->Get()->Record.MutableResponse()->mutable_operation()->mutable_result()->PackFrom(listTenantsResult); break; } case TEvWhiteboard::EvBSGroupStateResponse: { auto *x = reinterpret_cast(&ev); - Cerr << "aaaa EvBSGroupStateResponse 1" << Endl; ChangeBSGroupStateResponse(x); - Cerr << "aaaa EvBSGroupStateResponse 2" << Endl; break; } case TEvWhiteboard::EvVDiskStateResponse: { auto *x = reinterpret_cast(&ev); - Cerr << "aaaa EvVDiskStateResponse 1" << Endl; ChangeVDiskStateOn9NodeResponse(x); - Cerr << "aaaa EvVDiskStateResponse 2" << Endl; break; } case TEvWhiteboard::EvPDiskStateResponse: { auto *x = reinterpret_cast(&ev); - Cerr << "aaaa EvPDiskStateResponse 1" << Endl; ChangePDiskStateResponse(x); - Cerr << "aaaa EvPDiskStateResponse 2" << Endl; + break; + } + case TEvBlobStorage::EvControllerSelectGroupsResult: { + auto *x = reinterpret_cast(&ev); + AddGroupsInControllerSelectGroupsResult(x, 9); break; } } @@ -1211,7 +1251,6 @@ Y_UNIT_TEST_SUITE(Viewer) { size_t pos = result->Answer.find('{'); TString jsonResult = result->Answer.substr(pos); - Cerr << "json result: " << jsonResult << Endl; NJson::TJsonValue json; try { NJson::ReadJsonTree(jsonResult, &json, true); @@ -1219,10 +1258,40 @@ Y_UNIT_TEST_SUITE(Viewer) { catch (yexception ex) { Ctest << ex.what() << Endl; } - // UNIT_ASSERT_VALUES_EQUAL(json.GetMap().contains("StorageGroups"), isExpectingGroup); + + UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundGroups"), ToString(expectedFoundGroups)); + UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalGroups"), ToString(expectedTotalGroups)); + } + + Y_UNIT_TEST(JsonStorageListingV1) { + JsonStorage9Nodes9GroupsListingTest("v1", false, false, false, 9, 9); + } + + Y_UNIT_TEST(JsonStorageListingV2) { + JsonStorage9Nodes9GroupsListingTest("v2", false, false, false, 9, 9); + } + + Y_UNIT_TEST(JsonStorageListingV1GroupIdFilter) { + JsonStorage9Nodes9GroupsListingTest("v1", true, false, false, 1, 9); + } + + Y_UNIT_TEST(JsonStorageListingV2GroupIdFilter) { + JsonStorage9Nodes9GroupsListingTest("v2", true, false, false, 1, 9); + } + + Y_UNIT_TEST(JsonStorageListingV1NodeIdFilter) { + JsonStorage9Nodes9GroupsListingTest("v1", false, true, false, 8, 8); + } + + Y_UNIT_TEST(JsonStorageListingV2NodeIdFilter) { + JsonStorage9Nodes9GroupsListingTest("v2", false, true, false, 8, 8); + } + + Y_UNIT_TEST(JsonStorageListingV1PDiskIdFilter) { + JsonStorage9Nodes9GroupsListingTest("v1", false, true, true, 4, 8); } - Y_UNIT_TEST(JsonStorage9NodesListingV1NodeIdFilter) { - JsonStorage9NodesListingTest(); + Y_UNIT_TEST(JsonStorageListingV2PDiskIdFilter) { + JsonStorage9Nodes9GroupsListingTest("v2", false, true, true, 4, 8); } }