Skip to content

Commit

Permalink
Merge eca636e into 3a7aa37
Browse files Browse the repository at this point in the history
  • Loading branch information
@StekPerepolnen
StekPerepolnen authored May 2, 2024
2 parents 3a7aa37 + eca636e commit 29f2113
Show file tree
Hide file tree
Showing 5 changed files with 209 additions and 8 deletions.
4 changes: 4 additions & 0 deletions ydb/core/testlib/actors/test_runtime.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,10 @@ namespace NActors {
return *node->GetAppData<NKikimr::TAppData>();
}

ui32 TTestActorRuntime::GetFirstNodeId() {
return FirstNodeId;
}

bool TTestActorRuntime::DefaultScheduledFilterFunc(TTestActorRuntimeBase& runtime, TAutoPtr<IEventHandle>& event, TDuration delay, TInstant& deadline) {
Y_UNUSED(delay);
Y_UNUSED(deadline);
Expand Down
1 change: 1 addition & 0 deletions ydb/core/testlib/actors/test_runtime.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ namespace NActors {
void ClosePipe(TActorId clientId, const TActorId& sender, ui32 nodeIndex);
void DisconnectNodes(ui32 fromNodeIndex, ui32 toNodeIndex, bool async = true);
NKikimr::TAppData& GetAppData(ui32 nodeIndex = 0);
ui32 GetFirstNodeId();

TPortManager& GetPortManager() {
return *this;
Expand Down
3 changes: 2 additions & 1 deletion ydb/core/viewer/json_storage.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ class TJsonStorage : public TJsonStorageBase {
}

bool CheckGroupFilters(const TString& groupId, const TString& poolName, const TGroupRow& groupRow) {
if (!EffectiveFilterGroupIds.empty() && !EffectiveFilterGroupIds.contains(groupId)) {
if (!EffectiveGroupFilter.contains(groupId)) {
return false;
}
switch (With) {
Expand Down Expand Up @@ -497,6 +497,7 @@ struct TJsonRequestParameters<TJsonStorage> {
{"name":"tenant","in":"query","description":"tenant name","required":false,"type":"string"},
{"name":"pool","in":"query","description":"storage pool name","required":false,"type":"string"},
{"name":"node_id","in":"query","description":"node id","required":false,"type":"integer"},
{"name":"pdisk_id","in":"query","description":"pdisk id","required":false,"type":"integer"},
{"name":"group_id","in":"query","description":"group id","required":false,"type":"integer"},
{"name":"need_groups","in":"query","description":"return groups information","required":false,"type":"boolean","default":true},
{"name":"need_disks","in":"query","description":"return disks information","required":false,"type":"boolean","default":true},
Expand Down
19 changes: 12 additions & 7 deletions ydb/core/viewer/json_storage_base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,11 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
ui32 Timeout = 0;
TString FilterTenant;
THashSet<TString> FilterStoragePools;
TVector<TString> FilterGroupIds;
TString Filter;
std::unordered_set<TString> FilterGroupIds;
std::unordered_set<TNodeId> FilterNodeIds;
THashSet<TString> EffectiveFilterGroupIds;
std::unordered_set<ui32> FilterPDiskIds;
THashSet<TString> EffectiveGroupFilter;
std::unordered_set<TNodeId> NodeIds;
bool NeedAdditionalNodesRequests;

Expand Down Expand Up @@ -133,9 +134,9 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
FilterStoragePools.emplace(filterStoragePool);
}
SplitIds(params.Get("node_id"), ',', FilterNodeIds);
SplitIds(params.Get("pdisk_id"), ',', FilterPDiskIds);
NeedAdditionalNodesRequests = !FilterNodeIds.empty();
SplitIds(params.Get("group_id"), ',', FilterGroupIds);
Sort(FilterGroupIds);
Filter = params.Get("filter");
if (params.Get("with") == "missing") {
With = EWith::MissingDisks;
Expand Down Expand Up @@ -356,6 +357,14 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
for (auto& vDiskStateInfo : *(vDiskInfo.MutableVDiskStateInfo())) {
vDiskStateInfo.SetNodeId(nodeId);
VDiskId2vDiskStateInfo[VDiskIDFromVDiskID(vDiskStateInfo.GetVDiskId())] = &vDiskStateInfo;

bool isNodeIdValid = FilterNodeIds.empty() || FilterNodeIds.contains(nodeId);
bool isPDiskIdValid = FilterNodeIds.empty() || FilterPDiskIds.empty() || FilterPDiskIds.contains(vDiskStateInfo.GetPDiskId());
bool isGroupIdValid = FilterGroupIds.empty() || FilterGroupIds.contains(ToString(vDiskStateInfo.GetVDiskId().GetGroupID()));

if (isNodeIdValid && isPDiskIdValid && isGroupIdValid) {
EffectiveGroupFilter.insert(ToString(vDiskStateInfo.GetVDiskId().GetGroupID()));
}
}
RequestDone();
}
Expand All @@ -375,10 +384,6 @@ class TJsonStorageBase : public TViewerPipeClient<TJsonStorageBase> {
}
if (FilterNodeIds.empty() || FilterNodeIds.contains(info.GetNodeId())) {
StoragePoolInfo[storagePoolName].Groups.emplace(ToString(info.GetGroupID()));
TString groupId(ToString(info.GetGroupID()));
if (FilterGroupIds.empty() || BinarySearch(FilterGroupIds.begin(), FilterGroupIds.end(), groupId)) {
EffectiveFilterGroupIds.insert(groupId);
}
}
for (const auto& vDiskNodeId : info.GetVDiskNodeIds()) {
Group2NodeId[info.GetGroupID()].push_back(vDiskNodeId);
Expand Down
190 changes: 190 additions & 0 deletions ydb/core/viewer/viewer_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1325,4 +1325,194 @@ Y_UNIT_TEST_SUITE(Viewer) {
"description",
});
}

void ChangeBSGroupStateResponse(TEvWhiteboard::TEvBSGroupStateResponse::TPtr* ev) {
ui64 nodeId = (*ev)->Cookie;
auto& pbRecord = (*ev)->Get()->Record;

pbRecord.clear_bsgroupstateinfo();

for (ui64 groupId = 1; groupId <= 9; groupId++) {
if (groupId % 9 == nodeId % 9) {
continue;
}
auto state = pbRecord.add_bsgroupstateinfo();
state->set_groupid(groupId);
state->set_storagepoolname("/Root:test");
state->set_nodeid(nodeId);
for (int k = 1; k <= 8; k++) {
auto vdisk = groupId * 8 + k;
auto vdiskId = state->add_vdiskids();
vdiskId->set_groupid(groupId);
vdiskId->set_groupgeneration(1);
vdiskId->set_vdisk(vdisk);
}
}
}

void ChangePDiskStateResponse(TEvWhiteboard::TEvPDiskStateResponse::TPtr* ev) {
auto& pbRecord = (*ev)->Get()->Record;
pbRecord.clear_pdiskstateinfo();
for (int k = 0; k < 2; k++) {
auto state = pbRecord.add_pdiskstateinfo();
state->set_pdiskid(k);
}
}

void ChangeVDiskStateOn9NodeResponse(NNodeWhiteboard::TEvWhiteboard::TEvVDiskStateResponse::TPtr* ev) {
ui64 nodeId = (*ev)->Cookie;
auto& pbRecord = (*ev)->Get()->Record;

pbRecord.clear_vdiskstateinfo();

for (int k = 0; k < 8; k++) {
auto groupId = (nodeId + k) % 9 + 1;
auto vdisk = groupId * 8 + k + 1;
ui32 pdisk = k / 4;
ui32 slotid = k % 4;
auto state = pbRecord.add_vdiskstateinfo();
state->set_pdiskid(pdisk);
state->set_vdiskslotid(slotid);
state->mutable_vdiskid()->set_groupid(groupId);
state->mutable_vdiskid()->set_groupgeneration(1);
state->mutable_vdiskid()->set_vdisk(vdisk++);
state->set_vdiskstate(NKikimrWhiteboard::EVDiskState::OK);
state->set_nodeid(nodeId);
}
}

void AddGroupsInControllerSelectGroupsResult(TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr* ev, int groupCount) {
auto& pbRecord = (*ev)->Get()->Record;
auto pbMatchGroups = pbRecord.mutable_matchinggroups(0);

auto sample = pbMatchGroups->groups(0);
pbMatchGroups->ClearGroups();

for (int groupId = 1; groupId <= groupCount; groupId++) {
auto group = pbMatchGroups->add_groups();
group->CopyFrom(sample);
group->set_groupid(groupId++);
group->set_storagepoolname("/Root:test");
}
};

void JsonStorage9Nodes9GroupsListingTest(TString version, bool groupFilter, bool nodeFilter, bool pdiskFilter, ui32 expectedFoundGroups, ui32 expectedTotalGroups) {
TPortManager tp;
ui16 port = tp.GetPort(2134);
ui16 grpcPort = tp.GetPort(2135);
auto settings = TServerSettings(port);
settings.InitKikimrRunConfig()
.SetNodeCount(9)
.SetUseRealThreads(false)
.SetDomainName("Root");
TServer server(settings);
server.EnableGRpc(grpcPort);
TClient client(settings);
TTestActorRuntime& runtime = *server.GetRuntime();

TActorId sender = runtime.AllocateEdgeActor();
TAutoPtr<IEventHandle> handle;

THttpRequest httpReq(HTTP_METHOD_GET);
httpReq.CgiParameters.emplace("with", "all");
httpReq.CgiParameters.emplace("version", version);
if (groupFilter) {
httpReq.CgiParameters.emplace("group_id", "1");
}
if (nodeFilter) {
httpReq.CgiParameters.emplace("node_id", ToString(runtime.GetFirstNodeId()));
}
if (pdiskFilter) {
httpReq.CgiParameters.emplace("pdisk_id", "0");
}
auto page = MakeHolder<TMonPage>("viewer", "title");
TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, page.Get(), "/json/storage", nullptr);
auto request = MakeHolder<NMon::TEvHttpInfo>(monReq);

auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
Y_UNUSED(ev);
switch (ev->GetTypeRewrite()) {
case NConsole::TEvConsole::EvListTenantsResponse: {
auto *x = reinterpret_cast<NConsole::TEvConsole::TEvListTenantsResponse::TPtr*>(&ev);
Ydb::Cms::ListDatabasesResult listTenantsResult;
(*x)->Get()->Record.GetResponse().operation().result().UnpackTo(&listTenantsResult);
listTenantsResult.Addpaths("/Root");
(*x)->Get()->Record.MutableResponse()->mutable_operation()->mutable_result()->PackFrom(listTenantsResult);
break;
}
case TEvWhiteboard::EvBSGroupStateResponse: {
auto *x = reinterpret_cast<TEvWhiteboard::TEvBSGroupStateResponse::TPtr*>(&ev);
ChangeBSGroupStateResponse(x);
break;
}
case TEvWhiteboard::EvVDiskStateResponse: {
auto *x = reinterpret_cast<TEvWhiteboard::TEvVDiskStateResponse::TPtr*>(&ev);
ChangeVDiskStateOn9NodeResponse(x);
break;
}
case TEvWhiteboard::EvPDiskStateResponse: {
auto *x = reinterpret_cast<TEvWhiteboard::TEvPDiskStateResponse::TPtr*>(&ev);
ChangePDiskStateResponse(x);
break;
}
case TEvBlobStorage::EvControllerSelectGroupsResult: {
auto *x = reinterpret_cast<TEvBlobStorage::TEvControllerSelectGroupsResult::TPtr*>(&ev);
AddGroupsInControllerSelectGroupsResult(x, 9);
break;
}
}

return TTestActorRuntime::EEventAction::PROCESS;
};
runtime.SetObserverFunc(observerFunc);

runtime.Send(new IEventHandle(NKikimr::NViewer::MakeViewerID(0), sender, request.Release(), 0));
NMon::TEvHttpInfoRes* result = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(handle);

size_t pos = result->Answer.find('{');
TString jsonResult = result->Answer.substr(pos);
NJson::TJsonValue json;
try {
NJson::ReadJsonTree(jsonResult, &json, true);
}
catch (yexception ex) {
Ctest << ex.what() << Endl;
}

UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("FoundGroups"), ToString(expectedFoundGroups));
UNIT_ASSERT_VALUES_EQUAL(json.GetMap().at("TotalGroups"), ToString(expectedTotalGroups));
}

Y_UNIT_TEST(JsonStorageListingV1) {
JsonStorage9Nodes9GroupsListingTest("v1", false, false, false, 9, 9);
}

Y_UNIT_TEST(JsonStorageListingV2) {
JsonStorage9Nodes9GroupsListingTest("v2", false, false, false, 9, 9);
}

Y_UNIT_TEST(JsonStorageListingV1GroupIdFilter) {
JsonStorage9Nodes9GroupsListingTest("v1", true, false, false, 1, 9);
}

Y_UNIT_TEST(JsonStorageListingV2GroupIdFilter) {
JsonStorage9Nodes9GroupsListingTest("v2", true, false, false, 1, 9);
}

Y_UNIT_TEST(JsonStorageListingV1NodeIdFilter) {
JsonStorage9Nodes9GroupsListingTest("v1", false, true, false, 8, 8);
}

Y_UNIT_TEST(JsonStorageListingV2NodeIdFilter) {
JsonStorage9Nodes9GroupsListingTest("v2", false, true, false, 8, 8);
}

Y_UNIT_TEST(JsonStorageListingV1PDiskIdFilter) {
JsonStorage9Nodes9GroupsListingTest("v1", false, true, true, 4, 8);
JsonStorage9Nodes9GroupsListingTest("v1", false, true, true, 4, 8);
}

Y_UNIT_TEST(JsonStorageListingV2PDiskIdFilter) {
JsonStorage9Nodes9GroupsListingTest("v2", false, true, true, 4, 8);
}
}

0 comments on commit 29f2113

Please sign in to comment.