Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

24-1: backport ydb#2397 add allow/deny lists to configs dispatcher #2790

Merged
merged 1 commit into from
Mar 15, 2024
Merged

24-1: backport ydb#2397 add allow/deny lists to configs dispatcher #2790

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ydb/core/cms/cms_ut_common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,13 @@ static void SetupServices(TTestActorRuntime &runtime, const TTestEnvOpts &option
NKikimrConfig::TAppConfig appConfig;
appConfig.MutableBootstrapConfig()->CopyFrom(TFakeNodeWhiteboardService::BootstrapConfig);
runtime.AddLocalService(MakeConfigsDispatcherID(runtime.GetNodeId(0)),
TActorSetupCmd(CreateConfigsDispatcher(appConfig, {}), TMailboxType::Simple, 0), 0);
TActorSetupCmd(CreateConfigsDispatcher(
NKikimr::NConsole::TConfigsDispatcherInitInfo {
.InitialConfig = appConfig,
}),
TMailboxType::Simple,
0),
0);

runtime.Initialize(app.Unwrap());
auto dnsConfig = new TDynamicNameserviceConfig();
Expand Down
131 changes: 75 additions & 56 deletions ydb/core/cms/console/configs_dispatcher.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,12 +133,7 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {
return NKikimrServices::TActivity::CONFIGS_DISPATCHER_ACTOR;
}

TConfigsDispatcher(
const NKikimrConfig::TAppConfig &config,
const TMap<TString, TString> &labels,
const NKikimrConfig::TAppConfig &initialCmsConfig,
const NKikimrConfig::TAppConfig &initialCmsYamlConfig,
const THashMap<ui32, TConfigItemInfo> &configInitInfo);
TConfigsDispatcher(const TConfigsDispatcherInitInfo& initInfo);

void Bootstrap();

Expand All @@ -163,6 +158,8 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {

NKikimrConfig::TAppConfig ParseYamlProtoConfig();

TDynBitMap FilterKinds(const TDynBitMap& in);

void Handle(NMon::TEvHttpInfo::TPtr &ev);
void Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev);
void Handle(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev);
Expand Down Expand Up @@ -230,12 +227,11 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {


private:
TMap<TString, TString> Labels;
const NKikimrConfig::TAppConfig InitialConfig;
const TMap<TString, TString> Labels;
const std::variant<std::monostate, TDenyList, TAllowList> ItemsServeRules;
const NKikimrConfig::TAppConfig BaseConfig;
NKikimrConfig::TAppConfig CurrentConfig;
const NKikimrConfig::TAppConfig InitialCmsConfig;
const NKikimrConfig::TAppConfig InitialCmsYamlConfig;
const THashMap<ui32, TConfigItemInfo> ConfigInitInfo;
const std::optional<TDebugInfo> DebugInfo;
ui64 NextRequestCookie;
TVector<TActorId> HttpRequests;
TActorId CommonSubscriptionClient;
Expand All @@ -255,21 +251,14 @@ class TConfigsDispatcher : public TActorBootstrapped<TConfigsDispatcher> {

};

TConfigsDispatcher::TConfigsDispatcher(
const NKikimrConfig::TAppConfig &config,
const TMap<TString, TString> &labels,
const NKikimrConfig::TAppConfig &initialCmsConfig,
const NKikimrConfig::TAppConfig &initialCmsYamlConfig,
const THashMap<ui32, TConfigItemInfo> &configInitInfo)
: Labels(labels)
, InitialConfig(config)
, CurrentConfig(config)
, InitialCmsConfig(initialCmsConfig)
, InitialCmsYamlConfig(initialCmsYamlConfig)
, ConfigInitInfo(configInitInfo)
TConfigsDispatcher::TConfigsDispatcher(const TConfigsDispatcherInitInfo& initInfo)
: Labels(initInfo.Labels)
, ItemsServeRules(initInfo.ItemsServeRules)
, BaseConfig(initInfo.InitialConfig)
, CurrentConfig(initInfo.InitialConfig)
, DebugInfo(initInfo.DebugInfo)
, NextRequestCookie(Now().GetValue())
{
}
{}

void TConfigsDispatcher::Bootstrap()
{
Expand Down Expand Up @@ -400,9 +389,12 @@ void TConfigsDispatcher::ReplyMonJson(TActorId mailbox) {
response.InsertValue("yaml_config", YamlConfig);
response.InsertValue("resolved_json_config", NJson::ReadJsonFastTree(ResolvedJsonConfig, true));
response.InsertValue("current_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(CurrentConfig, NYamlConfig::GetProto2JsonConfig()), true));
response.InsertValue("initial_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(InitialConfig, NYamlConfig::GetProto2JsonConfig()), true));
response.InsertValue("initial_cms_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(InitialCmsConfig, NYamlConfig::GetProto2JsonConfig()), true));
response.InsertValue("initial_cms_yaml_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(InitialCmsYamlConfig, NYamlConfig::GetProto2JsonConfig()), true));

if (DebugInfo) {
response.InsertValue("initial_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(DebugInfo->StaticConfig, NYamlConfig::GetProto2JsonConfig()), true));
response.InsertValue("initial_cms_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(DebugInfo->OldDynConfig, NYamlConfig::GetProto2JsonConfig()), true));
response.InsertValue("initial_cms_yaml_json_config", NJson::ReadJsonFastTree(NProtobufJson::Proto2Json(DebugInfo->NewDynConfig, NYamlConfig::GetProto2JsonConfig()), true));
}

NJson::WriteJson(&str, &response, {});

Expand All @@ -419,6 +411,29 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigNotificationRequest::TPtr &
Send(ev->Sender, resp.Release(), 0, ev->Cookie);
}


TDynBitMap TConfigsDispatcher::FilterKinds(const TDynBitMap& in) {
TDynBitMap out;

if (const auto* denyList = std::get_if<TDenyList>(&ItemsServeRules)) {
Y_FOR_EACH_BIT(kind, in) {
if (!denyList->Items.contains(kind)) {
out.Set(kind);
}
}
} else if (const auto* allowList = std::get_if<TAllowList>(&ItemsServeRules)) {
Y_FOR_EACH_BIT(kind, in) {
if (allowList->Items.contains(kind)) {
out.Set(kind);
}
}
} else {
out = in;
}

return out;
}

void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev)
{
Y_UNUSED(ev);
Expand Down Expand Up @@ -494,14 +509,14 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev)
str << "Coloring: \"<font color=\"red\">config not set</font>\","
<< " \"<font color=\"green\">config set in dynamic config</font>\", \"<font color=\"#007bff\">config set in static config</font>\"" << Endl;
str << "</div>" << Endl;
NHttp::OutputRichConfigHTML(str, InitialConfig, YamlProtoConfig, CurrentConfig, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled);
NHttp::OutputRichConfigHTML(str, BaseConfig, YamlProtoConfig, CurrentConfig, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("effective-startup-config", "Effective startup config") {
str << "<div class=\"alert alert-primary tab-left\" role=\"alert\">" << Endl;
str << "Some of these configs may be overwritten by dynamic ones." << Endl;
str << "</div>" << Endl;
NHttp::OutputConfigHTML(str, InitialConfig);
NHttp::OutputConfigHTML(str, BaseConfig);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("effective-dynamic-config", "Effective dynamic config") {
Expand All @@ -510,18 +525,26 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev)
str << "</div>" << Endl;
NKikimrConfig::TAppConfig trunc;
if (YamlConfigEnabled) {
ReplaceConfigItems(YamlProtoConfig, trunc, KindsToBitMap(DYNAMIC_KINDS), InitialConfig);
ReplaceConfigItems(CurrentConfig, trunc, KindsToBitMap(NON_YAML_KINDS), trunc, false);
ReplaceConfigItems(YamlProtoConfig, trunc, FilterKinds(KindsToBitMap(DYNAMIC_KINDS)), BaseConfig);
ReplaceConfigItems(CurrentConfig, trunc, FilterKinds(KindsToBitMap(NON_YAML_KINDS)), trunc, false);
} else {
ReplaceConfigItems(CurrentConfig, trunc, KindsToBitMap(DYNAMIC_KINDS), InitialConfig);
ReplaceConfigItems(CurrentConfig, trunc, FilterKinds(KindsToBitMap(DYNAMIC_KINDS)), BaseConfig);
}
NHttp::OutputConfigHTML(str, trunc);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("debug-info", "Debug info") {
DIV_CLASS("tab-left") {
COLLAPSED_REF_CONTENT("effective-config-debug-info", "Effective config debug info") {
NHttp::OutputConfigDebugInfoHTML(str, InitialConfig, YamlProtoConfig, CurrentConfig, ConfigInitInfo, DYNAMIC_KINDS, NON_YAML_KINDS, YamlConfigEnabled);
NHttp::OutputConfigDebugInfoHTML(
str,
BaseConfig,
YamlProtoConfig,
CurrentConfig,
{DebugInfo ? DebugInfo->InitInfo : THashMap<ui32, TConfigItemInfo>{}},
DYNAMIC_KINDS,
NON_YAML_KINDS,
YamlConfigEnabled);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("state", "State") {
Expand Down Expand Up @@ -651,15 +674,17 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev)
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("initial-config", "Initial config") {
NHttp::OutputConfigHTML(str, InitialConfig);
NHttp::OutputConfigHTML(str, BaseConfig);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") {
NHttp::OutputConfigHTML(str, InitialCmsConfig);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") {
NHttp::OutputConfigHTML(str, InitialCmsYamlConfig);
if (DebugInfo) {
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("initial-cms-config", "Initial CMS config") {
NHttp::OutputConfigHTML(str, DebugInfo->OldDynConfig);
}
str << "<br />" << Endl;
COLLAPSED_REF_CONTENT("initial-cms-yaml-config", "Initial CMS YAML config") {
NHttp::OutputConfigHTML(str, DebugInfo->NewDynConfig);
}
}
}
}
Expand Down Expand Up @@ -742,7 +767,7 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T
bool hasAffectedKinds = false;

if (subscription->Yaml && YamlConfigEnabled) {
ReplaceConfigItems(YamlProtoConfig, trunc, subscription->Kinds, InitialConfig);
ReplaceConfigItems(YamlProtoConfig, trunc, FilterKinds(subscription->Kinds), BaseConfig);
} else {
Y_FOR_EACH_BIT(kind, kinds) {
if (affectedKinds.contains(kind)) {
Expand All @@ -755,7 +780,7 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T
continue;
}

ReplaceConfigItems(ev->Get()->Record.GetConfig(), trunc, kinds, InitialConfig);
ReplaceConfigItems(ev->Get()->Record.GetConfig(), trunc, FilterKinds(kinds), BaseConfig);
}

if (hasAffectedKinds || !CompareConfigs(subscription->CurrentConfig.Config, trunc) || CurrentStateFunc() == &TThis::StateInit) {
Expand Down Expand Up @@ -820,9 +845,9 @@ void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvGetConfigRequest::TPtr
auto trunc = std::make_shared<NKikimrConfig::TAppConfig>();
auto kinds = KindsToBitMap(ev->Get()->ConfigItemKinds);
if (YamlConfigEnabled && yamlKinds) {
ReplaceConfigItems(YamlProtoConfig, *trunc, kinds, InitialConfig);
ReplaceConfigItems(YamlProtoConfig, *trunc, FilterKinds(kinds), BaseConfig);
} else {
ReplaceConfigItems(CurrentConfig, *trunc, kinds, InitialConfig);
ReplaceConfigItems(CurrentConfig, *trunc, FilterKinds(kinds), BaseConfig);
}
resp->Config = trunc;

Expand Down Expand Up @@ -893,9 +918,9 @@ void TConfigsDispatcher::Handle(TEvConfigsDispatcher::TEvSetConfigSubscriptionRe
subscription->UpdateInProcess = MakeHolder<TEvConsole::TEvConfigNotificationRequest>();
NKikimrConfig::TAppConfig trunc;
if (YamlConfigEnabled) {
ReplaceConfigItems(YamlProtoConfig, trunc, kinds, InitialConfig);
ReplaceConfigItems(YamlProtoConfig, trunc, FilterKinds(kinds), BaseConfig);
} else {
ReplaceConfigItems(CurrentConfig, trunc, kinds, InitialConfig);
ReplaceConfigItems(CurrentConfig, trunc, FilterKinds(kinds), BaseConfig);
}
subscription->UpdateInProcess->Record.MutableConfig()->CopyFrom(trunc);
Y_FOR_EACH_BIT(kind, kinds) {
Expand Down Expand Up @@ -997,14 +1022,8 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvGetNodeLabelsRequest::TPtr &ev) {
Send(ev->Sender, Response.Release());
}

IActor *CreateConfigsDispatcher(
const NKikimrConfig::TAppConfig &config,
const TMap<TString, TString> &labels,
const NKikimrConfig::TAppConfig &initialCmsConfig,
const NKikimrConfig::TAppConfig &initialCmsYamlConfig,
const THashMap<ui32, TConfigItemInfo> &configInitInfo)
{
return new TConfigsDispatcher(config, labels, initialCmsConfig, initialCmsYamlConfig, configInitInfo);
IActor *CreateConfigsDispatcher(const TConfigsDispatcherInitInfo& initInfo) {
return new TConfigsDispatcher(initInfo);
}

} // namespace NKikimr::NConsole
30 changes: 24 additions & 6 deletions ydb/core/cms/console/configs_dispatcher.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include <ydb/core/cms/console/config_item_info.h>

#include <util/generic/vector.h>
#include <util/generic/map.h>

namespace NKikimr::NConsole {

Expand Down Expand Up @@ -107,17 +108,34 @@ struct TEvConfigsDispatcher {
};
};

struct TDenyList {
std::set<ui32> Items;
};

struct TAllowList {
std::set<ui32> Items;
};

struct TDebugInfo {
NKikimrConfig::TAppConfig StaticConfig;
NKikimrConfig::TAppConfig OldDynConfig;
NKikimrConfig::TAppConfig NewDynConfig;
THashMap<ui32, TConfigItemInfo> InitInfo;
};

struct TConfigsDispatcherInitInfo {
NKikimrConfig::TAppConfig InitialConfig;
TMap<TString, TString> Labels;
std::variant<std::monostate, TDenyList, TAllowList> ItemsServeRules;
std::optional<TDebugInfo> DebugInfo;
};

/**
* Initial config is used to initilize Configs Dispatcher. All received configs
* are compared to the current one and notifications are not sent to local
* subscribers if there is no config modification detected.
*/
IActor *CreateConfigsDispatcher(
const NKikimrConfig::TAppConfig &config,
const TMap<TString, TString> &labels,
const NKikimrConfig::TAppConfig &initialCmsConfig = {},
const NKikimrConfig::TAppConfig &initialCmsYamlConfig = {},
const THashMap<ui32, TConfigItemInfo> &configInitInfo = {});
IActor *CreateConfigsDispatcher(const TConfigsDispatcherInitInfo& initInfo);

inline TActorId MakeConfigsDispatcherID(ui32 node = 0) {
char x[12] = { 'c', 'o', 'n', 'f', 'i', 'g', 's', 'd', 'i', 's', 'p' };
Expand Down
13 changes: 12 additions & 1 deletion ydb/core/driver_lib/run/kikimr_services_initializers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2381,7 +2381,18 @@ TConfigsDispatcherInitializer::TConfigsDispatcherInitializer(const TKikimrRunCon
}

void TConfigsDispatcherInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) {
IActor* actor = NConsole::CreateConfigsDispatcher(Config, Labels, InitialCmsConfig, InitialCmsYamlConfig, ConfigInitInfo);
NKikimr::NConsole::TConfigsDispatcherInitInfo initInfo {
.InitialConfig = Config,
.Labels = Labels,
.ItemsServeRules = std::monostate{},
.DebugInfo = NKikimr::NConsole::TDebugInfo {
.StaticConfig = Config,
.OldDynConfig = InitialCmsConfig,
.NewDynConfig = InitialCmsYamlConfig,
.InitInfo = ConfigInitInfo,
},
};
IActor* actor = NConsole::CreateConfigsDispatcher(initInfo);
setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(
NConsole::MakeConfigsDispatcherID(NodeId),
TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)));
Expand Down
7 changes: 6 additions & 1 deletion ydb/core/testlib/tenant_runtime.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1034,7 +1034,12 @@ void TTenantTestRuntime::Setup(bool createTenantPools)
labels[label.GetName()] = label.GetValue();
}
labels.emplace("node_id", ToString(i));
auto aid = Register(CreateConfigsDispatcher(Extension, labels));
auto aid = Register(CreateConfigsDispatcher(
NKikimr::NConsole::TConfigsDispatcherInitInfo {
.InitialConfig = Extension,
.Labels = labels,
}
));
EnableScheduleForActor(aid, true);
RegisterService(MakeConfigsDispatcherID(GetNodeId(0)), aid, 0);
}
Expand Down
5 changes: 4 additions & 1 deletion ydb/core/testlib/test_client.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -739,7 +739,10 @@ namespace Tests {
if (!initial.HasImmediateControlsConfig()) {
initial.MutableImmediateControlsConfig()->CopyFrom(Settings->Controls);
}
auto *dispatcher = NConsole::CreateConfigsDispatcher(initial, {});
auto *dispatcher = NConsole::CreateConfigsDispatcher(
NKikimr::NConsole::TConfigsDispatcherInitInfo {
.InitialConfig = initial,
});
auto aid = Runtime->Register(dispatcher, nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0);
Runtime->RegisterService(NConsole::MakeConfigsDispatcherID(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx);
}
Expand Down
Loading