diff --git a/pkg/runtime/application_config_provider.go b/pkg/runtime/application_config_provider.go index b81963946..cba275d2e 100644 --- a/pkg/runtime/application_config_provider.go +++ b/pkg/runtime/application_config_provider.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" + "github.com/flyteorg/flyteadmin/pkg/common" "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/config" "github.com/flyteorg/flytestdlib/logger" @@ -18,18 +19,57 @@ const notifications = "notifications" const domains = "domains" const externalEvents = "externalEvents" -var databaseConfig = config.MustRegisterSection(database, &interfaces.DbConfigSection{}) +const postgres = "postgres" -var defaultFlyteAdminConfig = &interfaces.ApplicationConfig{ +var databaseConfig = config.MustRegisterSection(database, &interfaces.DbConfigSection{ + Port: 5432, + User: postgres, + Host: postgres, + DbName: postgres, + ExtraOptions: "sslmode=disable", +}) +var flyteAdminConfig = config.MustRegisterSection(flyteAdmin, &interfaces.ApplicationConfig{ + ProfilerPort: 10254, + MetricsScope: "flyte:", + MetadataStoragePrefix: []string{"metadata", "admin"}, + EventVersion: 1, AsyncEventsBufferSize: 100, -} -var flyteAdminConfig = config.MustRegisterSection(flyteAdmin, defaultFlyteAdminConfig) - -var schedulerConfig = config.MustRegisterSection(scheduler, &interfaces.SchedulerConfig{}) -var remoteDataConfig = config.MustRegisterSection(remoteData, &interfaces.RemoteDataConfig{}) -var notificationsConfig = config.MustRegisterSection(notifications, &interfaces.NotificationsConfig{}) -var domainsConfig = config.MustRegisterSection(domains, &interfaces.DomainsConfig{}) -var externalEventsConfig = config.MustRegisterSection(externalEvents, &interfaces.ExternalEventsConfig{}) +}) +var schedulerConfig = config.MustRegisterSection(scheduler, &interfaces.SchedulerConfig{ + EventSchedulerConfig: interfaces.EventSchedulerConfig{ + Scheme: common.Local, + }, + WorkflowExecutorConfig: interfaces.WorkflowExecutorConfig{ + Scheme: common.Local, + }, +}) +var remoteDataConfig = config.MustRegisterSection(remoteData, &interfaces.RemoteDataConfig{ + Scheme: common.Local, + Region: "us-east-1", + SignedURL: interfaces.SignedURL{ + DurationMinutes: 3, + }, +}) +var notificationsConfig = config.MustRegisterSection(notifications, &interfaces.NotificationsConfig{ + Type: common.Local, +}) +var domainsConfig = config.MustRegisterSection(domains, &interfaces.DomainsConfig{ + { + ID: "development", + Name: "development", + }, + { + ID: "staging", + Name: "staging", + }, + { + ID: "production", + Name: "production", + }, +}) +var externalEventsConfig = config.MustRegisterSection(externalEvents, &interfaces.ExternalEventsConfig{ + Type: common.Local, +}) // Implementation of an interfaces.ApplicationConfiguration type ApplicationConfigurationProvider struct{} diff --git a/pkg/runtime/cluster_resource_provider.go b/pkg/runtime/cluster_resource_provider.go index 149f33d33..9d5b064e8 100644 --- a/pkg/runtime/cluster_resource_provider.go +++ b/pkg/runtime/cluster_resource_provider.go @@ -1,55 +1,39 @@ package runtime import ( - "context" "time" "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/config" - "github.com/flyteorg/flytestdlib/logger" ) const clusterResourceKey = "cluster_resources" -var clusterResourceConfig = config.MustRegisterSection(clusterResourceKey, &interfaces.ClusterResourceConfig{}) +var clusterResourceConfig = config.MustRegisterSection(clusterResourceKey, &interfaces.ClusterResourceConfig{ + TemplateData: make(map[string]interfaces.DataSource), + RefreshInterval: config.Duration{ + Duration: time.Minute, + }, + CustomData: make(map[interfaces.DomainName]interfaces.TemplateData), +}) // Implementation of an interfaces.ClusterResourceConfiguration type ClusterResourceConfigurationProvider struct{} func (p *ClusterResourceConfigurationProvider) GetTemplatePath() string { - if clusterResourceConfig != nil && clusterResourceConfig.GetConfig() != nil { - return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).TemplatePath - } - logger.Warningf(context.Background(), - "Failed to find cluster resource values in config. Returning an empty string for template path") - return "" + return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).TemplatePath } func (p *ClusterResourceConfigurationProvider) GetTemplateData() interfaces.TemplateData { - if clusterResourceConfig != nil && clusterResourceConfig.GetConfig() != nil { - return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).TemplateData - } - logger.Warningf(context.Background(), - "Failed to find cluster resource values in config. Returning an empty map for template data") - return make(interfaces.TemplateData) + return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).TemplateData } func (p *ClusterResourceConfigurationProvider) GetRefreshInterval() time.Duration { - if clusterResourceConfig != nil && clusterResourceConfig.GetConfig() != nil { - return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).RefreshInterval.Duration - } - logger.Warningf(context.Background(), - "Failed to find cluster resource values in config. Returning 1 minute for refresh interval") - return time.Minute + return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).RefreshInterval.Duration } func (p *ClusterResourceConfigurationProvider) GetCustomTemplateData() map[interfaces.DomainName]interfaces.TemplateData { - if clusterResourceConfig != nil && clusterResourceConfig.GetConfig() != nil { - return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).CustomData - } - logger.Warningf(context.Background(), - "Failed to find cucluster resource values in config. Returning an empty map for custom template data") - return make(map[interfaces.DomainName]interfaces.TemplateData) + return clusterResourceConfig.GetConfig().(*interfaces.ClusterResourceConfig).CustomData } func NewClusterResourceConfigurationProvider() interfaces.ClusterResourceConfiguration { diff --git a/pkg/runtime/execution_queue_provider.go b/pkg/runtime/execution_queue_provider.go index bced2d9f6..f4e5a566d 100644 --- a/pkg/runtime/execution_queue_provider.go +++ b/pkg/runtime/execution_queue_provider.go @@ -1,36 +1,27 @@ package runtime import ( - "context" - "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" - "github.com/flyteorg/flytestdlib/logger" - "github.com/flyteorg/flytestdlib/config" ) const queuesKey = "queues" -var executionQueuesConfig = config.MustRegisterSection(queuesKey, &interfaces.QueueConfig{}) +var executionQueuesConfig = config.MustRegisterSection(queuesKey, &interfaces.QueueConfig{ + ExecutionQueues: make([]interfaces.ExecutionQueue, 0), + WorkflowConfigs: make([]interfaces.WorkflowConfig, 0), +}) // Implementation of an interfaces.QueueConfiguration type QueueConfigurationProvider struct{} func (p *QueueConfigurationProvider) GetExecutionQueues() []interfaces.ExecutionQueue { - if executionQueuesConfig != nil { - return executionQueuesConfig.GetConfig().(*interfaces.QueueConfig).ExecutionQueues - } - logger.Warningf(context.Background(), "Failed to find execution queues in config. Returning an empty slice") - return make([]interfaces.ExecutionQueue, 0) + return executionQueuesConfig.GetConfig().(*interfaces.QueueConfig).ExecutionQueues } func (p *QueueConfigurationProvider) GetWorkflowConfigs() []interfaces.WorkflowConfig { - if executionQueuesConfig != nil { - return executionQueuesConfig.GetConfig().(*interfaces.QueueConfig).WorkflowConfigs - } - logger.Warningf(context.Background(), "Failed to find workflows with attributes in config. Returning an empty slice") - return make([]interfaces.WorkflowConfig, 0) + return executionQueuesConfig.GetConfig().(*interfaces.QueueConfig).WorkflowConfigs } func NewQueueConfigurationProvider() interfaces.QueueConfiguration { diff --git a/pkg/runtime/namespace_config_provider.go b/pkg/runtime/namespace_config_provider.go index 7e07aab1f..931b6e51b 100644 --- a/pkg/runtime/namespace_config_provider.go +++ b/pkg/runtime/namespace_config_provider.go @@ -16,7 +16,9 @@ const ( projectDomainVariable = "project-domain" ) -var namespaceMappingConfig = config.MustRegisterSection(namespaceMappingKey, &interfaces.NamespaceMappingConfig{}) +var namespaceMappingConfig = config.MustRegisterSection(namespaceMappingKey, &interfaces.NamespaceMappingConfig{ + Mapping: projectDomainVariable, +}) type NamespaceMappingConfigurationProvider struct{} diff --git a/pkg/runtime/quality_of_service_provider.go b/pkg/runtime/quality_of_service_provider.go index dec15f88a..b61144652 100644 --- a/pkg/runtime/quality_of_service_provider.go +++ b/pkg/runtime/quality_of_service_provider.go @@ -11,7 +11,10 @@ import ( const qualityOfServiceKey = "qualityOfService" -var qualityOfServiceConfig = config.MustRegisterSection(qualityOfServiceKey, &interfaces.QualityOfServiceConfig{}) +var qualityOfServiceConfig = config.MustRegisterSection(qualityOfServiceKey, &interfaces.QualityOfServiceConfig{ + TierExecutionValues: make(map[interfaces.TierName]interfaces.QualityOfServiceSpec), + DefaultTiers: make(map[interfaces.DomainName]interfaces.TierName), +}) // Implementation of an interfaces.QualityOfServiceConfiguration type QualityOfServiceConfigProvider struct { @@ -19,25 +22,21 @@ type QualityOfServiceConfigProvider struct { func (p *QualityOfServiceConfigProvider) GetTierExecutionValues() map[core.QualityOfService_Tier]core.QualityOfServiceSpec { tierExecutionValues := make(map[core.QualityOfService_Tier]core.QualityOfServiceSpec) - if qualityOfServiceConfig != nil { - values := qualityOfServiceConfig.GetConfig().(*interfaces.QualityOfServiceConfig).TierExecutionValues - for tierName, spec := range values { - tierExecutionValues[core.QualityOfService_Tier(core.QualityOfService_Tier_value[tierName])] = - core.QualityOfServiceSpec{ - QueueingBudget: ptypes.DurationProto(spec.QueueingBudget.Duration), - } - } + configValues := qualityOfServiceConfig.GetConfig().(*interfaces.QualityOfServiceConfig).TierExecutionValues + for tierName, spec := range configValues { + tierExecutionValues[core.QualityOfService_Tier(core.QualityOfService_Tier_value[tierName])] = + core.QualityOfServiceSpec{ + QueueingBudget: ptypes.DurationProto(spec.QueueingBudget.Duration), + } } return tierExecutionValues } func (p *QualityOfServiceConfigProvider) GetDefaultTiers() map[interfaces.DomainName]core.QualityOfService_Tier { defaultTiers := make(map[interfaces.DomainName]core.QualityOfService_Tier) - if qualityOfServiceConfig != nil { - tiers := qualityOfServiceConfig.GetConfig().(*interfaces.QualityOfServiceConfig).DefaultTiers - for domainName, tierName := range tiers { - defaultTiers[domainName] = core.QualityOfService_Tier(core.QualityOfService_Tier_value[tierName]) - } + tiers := qualityOfServiceConfig.GetConfig().(*interfaces.QualityOfServiceConfig).DefaultTiers + for domainName, tierName := range tiers { + defaultTiers[domainName] = core.QualityOfService_Tier(core.QualityOfService_Tier_value[tierName]) } return defaultTiers } diff --git a/pkg/runtime/registration_validation_provider.go b/pkg/runtime/registration_validation_provider.go index 327feb342..d32050a5c 100644 --- a/pkg/runtime/registration_validation_provider.go +++ b/pkg/runtime/registration_validation_provider.go @@ -2,50 +2,33 @@ package runtime import ( - "context" - "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/config" - "github.com/flyteorg/flytestdlib/logger" ) const registration = "registration" -var registrationValidationConfig = config.MustRegisterSection(registration, &interfaces.RegistrationValidationConfig{}) +var registrationValidationConfig = config.MustRegisterSection(registration, &interfaces.RegistrationValidationConfig{ + MaxWorkflowNodes: 100, +}) // Implementation of an interfaces.TaskResourceConfiguration type RegistrationValidationProvider struct{} func (p *RegistrationValidationProvider) GetWorkflowNodeLimit() int { - if registrationValidationConfig != nil { - return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxWorkflowNodes - } - logger.Warning(context.Background(), "failed to find max workflow node values in config. Returning 0") - return 0 + return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxWorkflowNodes } func (p *RegistrationValidationProvider) GetMaxLabelEntries() int { - if registrationValidationConfig != nil { - return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxLabelEntries - } - logger.Warning(context.Background(), "failed to find max label entries in config. Returning 0") - return 0 + return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxLabelEntries } func (p *RegistrationValidationProvider) GetMaxAnnotationEntries() int { - if registrationValidationConfig != nil { - return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxAnnotationEntries - } - logger.Warning(context.Background(), "failed to find max annotation entries in config. Returning 0") - return 0 + return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).MaxAnnotationEntries } func (p *RegistrationValidationProvider) GetWorkflowSizeLimit() string { - if registrationValidationConfig != nil { - return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).WorkflowSizeLimit - } - logger.Warning(context.Background(), "failed to findworkflow size limit in config. Returning ''") - return "" + return registrationValidationConfig.GetConfig().(*interfaces.RegistrationValidationConfig).WorkflowSizeLimit } func NewRegistrationValidationProvider() interfaces.RegistrationValidationConfiguration { diff --git a/pkg/runtime/task_resource_provider.go b/pkg/runtime/task_resource_provider.go index fe22c905d..62649203e 100644 --- a/pkg/runtime/task_resource_provider.go +++ b/pkg/runtime/task_resource_provider.go @@ -1,11 +1,8 @@ package runtime import ( - "context" - "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/config" - "github.com/flyteorg/flytestdlib/logger" ) const taskResourceKey = "task_resources" @@ -21,19 +18,11 @@ type TaskResourceSpec struct { type TaskResourceProvider struct{} func (p *TaskResourceProvider) GetDefaults() interfaces.TaskResourceSet { - if taskResourceConfig != nil { - return taskResourceConfig.GetConfig().(*TaskResourceSpec).Defaults - } - logger.Warning(context.Background(), "failed to find task resource values in config. Returning empty struct") - return interfaces.TaskResourceSet{} + return taskResourceConfig.GetConfig().(*TaskResourceSpec).Defaults } func (p *TaskResourceProvider) GetLimits() interfaces.TaskResourceSet { - if taskResourceConfig != nil { - return taskResourceConfig.GetConfig().(*TaskResourceSpec).Limits - } - logger.Warning(context.Background(), "failed to find task resource values in config. Returning empty struct") - return interfaces.TaskResourceSet{} + return taskResourceConfig.GetConfig().(*TaskResourceSpec).Limits } func NewTaskResourceProvider() interfaces.TaskResourceConfiguration { diff --git a/pkg/runtime/whitelist_provider.go b/pkg/runtime/whitelist_provider.go index a363ddd8b..05fd6aae1 100644 --- a/pkg/runtime/whitelist_provider.go +++ b/pkg/runtime/whitelist_provider.go @@ -1,27 +1,22 @@ package runtime import ( - "context" - "github.com/flyteorg/flyteadmin/pkg/runtime/interfaces" "github.com/flyteorg/flytestdlib/config" - "github.com/flyteorg/flytestdlib/logger" ) const whitelistKey = "task_type_whitelist" -var whitelistConfig = config.MustRegisterSection(whitelistKey, &interfaces.TaskTypeWhitelist{}) +var whiteListProviderDefault = make(map[string][]interfaces.WhitelistScope) + +var whitelistConfig = config.MustRegisterSection(whitelistKey, &whiteListProviderDefault) // Implementation of an interfaces.QueueConfiguration type WhitelistConfigurationProvider struct{} func (p *WhitelistConfigurationProvider) GetTaskTypeWhitelist() interfaces.TaskTypeWhitelist { - if whitelistConfig != nil && whitelistConfig.GetConfig() != nil { - whitelists := whitelistConfig.GetConfig().(*interfaces.TaskTypeWhitelist) - return *whitelists - } - logger.Warningf(context.Background(), "Failed to find task type whitelist in config. Returning an empty slice") - return interfaces.TaskTypeWhitelist{} + whitelists := whitelistConfig.GetConfig().(*interfaces.TaskTypeWhitelist) + return *whitelists } func NewWhitelistConfigurationProvider() interfaces.WhitelistConfiguration {