revert: make agent starting period configurable [DET-3219] (#623)

This reverts commit 7f83e97.

deploy/determined_deploy/aws/cli.py

@@ -63,9 +63,6 @@ def make_up_subparser(subparsers: argparse._SubParsersAction):
     subparser.add_argument(
         "--max-idle-agent-period", type=str, help="max agent idle time",
     )
-    subparser.add_argument(
-        "--max-agent-starting-period", type=str, help="max agent starting time",
-    )
     subparser.add_argument(
         "--max-dynamic-agents",
         type=int,

deploy/determined_deploy/aws/constants.py

@@ -34,7 +34,6 @@ class cloudformation:
     BOTO3_SESSION = "Boto3Session"
     AGENT_TAG_NAME = "AgentTagName"
     MAX_IDLE_AGENT_PERIOD = "MaxIdleAgentPeriod"
-    MAX_AGENT_STARTING_PERIOD = "MaxAgentStartingPeriod"
     MAX_DYNAMIC_AGENTS = "MaxDynamicAgents"
     LOG_GROUP = "LogGroup"
     REGION = "Region"

deploy/determined_deploy/aws/deployment_types/secure.py

@@ -32,7 +32,6 @@ class Secure(base.DeterminedDeployment):
         constants.cloudformation.VERSION,
         constants.cloudformation.DB_PASSWORD,
         constants.cloudformation.MAX_IDLE_AGENT_PERIOD,
-        constants.cloudformation.MAX_AGENT_STARTING_PERIOD,
         constants.cloudformation.MAX_DYNAMIC_AGENTS,
     ]
 

deploy/determined_deploy/aws/deployment_types/simple.py

@@ -26,7 +26,6 @@ class Simple(base.DeterminedDeployment):
         constants.cloudformation.VERSION,
         constants.cloudformation.DB_PASSWORD,
         constants.cloudformation.MAX_IDLE_AGENT_PERIOD,
-        constants.cloudformation.MAX_AGENT_STARTING_PERIOD,
         constants.cloudformation.MAX_DYNAMIC_AGENTS,
     ]
 

deploy/determined_deploy/aws/deployment_types/vpc.py

@@ -25,7 +25,6 @@ class VPC(base.DeterminedDeployment):
         constants.cloudformation.VERSION,
         constants.cloudformation.DB_PASSWORD,
         constants.cloudformation.MAX_IDLE_AGENT_PERIOD,
-        constants.cloudformation.MAX_AGENT_STARTING_PERIOD,
         constants.cloudformation.MAX_DYNAMIC_AGENTS,
     ]
 

deploy/determined_deploy/aws/templates/secure.yaml

@@ -71,11 +71,6 @@ Parameters:
     Description: How long before idle agents are shutdown
     Default: 10m
 
-  MaxAgentStartingPeriod:
-    Type: String
-    Decription: How long for agent starting before retrying
-    Default: 10m
-
   MaxDynamicAgents:
     Type: Number
     Description: Maximum number of agents to launch simultaneously
@@ -591,7 +586,6 @@ Resources:
               log_stream: determined-agent
               master_url: http://local-ipv4:8080
               max_idle_agent_period: ${MaxIdleAgentPeriod}
-              max_agent_starting_period: ${MaxAgentStartingPeriod}
               max_instances: ${MaxDynamicAgents}
               network_interface:
                 public_ip: false

deploy/determined_deploy/aws/templates/simple.yaml

@@ -45,11 +45,6 @@ Parameters:
     Description: How long before idle agents are shutdown
     Default: 10m
 
-  MaxAgentStartingPeriod:
-    Type: String
-    Decription: How long for agent starting before retrying
-    Default: 10m
-
   MaxDynamicAgents:
     Type: Number
     Description: Maximum number of agents to launch simultaneously
@@ -387,7 +382,6 @@ Resources:
               log_stream: determined-agent
               master_url: http://local-ipv4:8080
               max_idle_agent_period: ${MaxIdleAgentPeriod}
-              max_agent_starting_period: ${MaxAgentStartingPeriod}
               max_instances: ${MaxDynamicAgents}
               network_interface:
                 public_ip: true

deploy/determined_deploy/aws/templates/vpc.yaml

@@ -59,11 +59,6 @@ Parameters:
     Description: How long before idle agents are shutdown
     Default: 10m
 
-  MaxAgentStartingPeriod:
-    Type: String
-    Decription: How long for agent starting before retrying
-    Default: 10m
-
   MaxDynamicAgents:
     Type: Number
     Description: Maximum number of agents to launch simultaneously
@@ -513,7 +508,6 @@ Resources:
               log_stream: determined-agent
               master_url: http://local-ipv4:8080
               max_idle_agent_period: ${MaxIdleAgentPeriod}
-              max_agent_starting_period: ${MaxAgentStartingPeriod}
               max_instances: ${MaxDynamicAgents}
               network_interface:
                 public_ip: true

deploy/determined_deploy/gcp/cli.py

@@ -117,12 +117,6 @@ def make_up_subparser(subparsers: argparse._SubParsersAction) -> None:
         default=constants.defaults.MAX_IDLE_AGENT_PERIOD,
         help="max agent idle time before it is shut down, e.g. 30m for 30 minutes",
     )
-    optional_named.add_argument(
-        "--max-agent-starting-period",
-        type=str,
-        default=constants.defaults.MAX_AGENT_STARTING_PERIOD,
-        help="max agent starting time before retrying, e.g. 30m for 30 minutes",
-    )
     optional_named.add_argument(
         "--port",
         type=int,

deploy/determined_deploy/gcp/constants.py

@@ -7,7 +7,6 @@ class defaults:
     GPU_TYPE = "nvidia-tesla-k80"
     MASTER_INSTANCE_TYPE = "n1-standard-2"
     MAX_IDLE_AGENT_PERIOD = "10m"
-    MAX_AGENT_STARTING_PERIOD = "10m"
     OPERATION_TIMEOUT_PERIOD = "5m"
     MAX_DYNAMIC_AGENTS = 5
     STATIC_AGENTS = 0

deploy/determined_deploy/gcp/terraform/main.tf

@@ -125,7 +125,6 @@ module "compute" {
   agent_docker_network = var.agent_docker_network
   agent_instance_type = var.agent_instance_type
   max_idle_agent_period = var.max_idle_agent_period
-  max_agent_starting_period: var.max_agent_starting_period
   gpu_type = var.gpu_type
   gpu_num = var.gpu_num
   max_dynamic_agents = var.max_dynamic_agents

deploy/determined_deploy/gcp/terraform/modules/compute/main.tf

@@ -43,7 +43,6 @@ resource "google_compute_instance" "master_instance" {
       master_url: ${var.scheme}://internal-ip:${var.port}
       agent_docker_network: ${var.agent_docker_network}
       max_idle_agent_period: ${var.max_idle_agent_period}
-      max_agent_starting_period: ${var.max_agent_starting_period}
       provider: gcp
       name_prefix: det-dynamic-agent-${var.unique_id}-
       network_interface:

deploy/determined_deploy/gcp/terraform/modules/compute/variables.tf

@@ -40,9 +40,6 @@ variable "agent_docker_network" {
 variable "max_idle_agent_period" {
 }
 
-variable "max_agent_starting_period" {
-}
-
 variable "tag_master_port" {
 }
 

deploy/determined_deploy/gcp/terraform/variables.tf

@@ -105,11 +105,6 @@ variable "max_idle_agent_period" {
   default = "10m"
 }
 
-variable "max_agent_starting_period" {
-  type = string
-  default = "10m"
-}
-
 variable "min_cpu_platform_master" {
   type = string
   default = "Intel Skylake"

docs/reference/cluster-config.txt

@@ -274,11 +274,6 @@ The master supports the following configuration settings:
     with optional fraction and a unit suffix, such as "30s", "1h", or
     "1m30s". Valid time units are "s", "m", "h". The default value is ``5m``.
 
-  - ``max_agent_starting_period``: How long to wait for agents starting before
-    retrying. This string is a sequence of decimal numbers, each
-    with optional fraction and a unit suffix, such as "30s", "1h", or
-    "1m30s". Valid time units are "s", "m", "h". The default value is ``5m``.
-
   - ``provider: aws``: Specifies running dynamic agents on AWS.
     (*Required*)
 

docs/topic-guides/elastic-infra/dynamic-agents-aws.txt

@@ -126,10 +126,8 @@ an example configuration. See :ref:`cluster-configuration` for details.
     master_url: <scheme://host:port>
     startup_script: <startup script>
     container_startup_script: <container startup script>
-    agent_docker_network: host
-    agent_docker_image: <agent docker image>
+    agent_docker_network: determined
     max_idle_agent_period: 5m
-    max_agent_starting_period: 5m
 
     provider: aws
     region: <region>

docs/topic-guides/elastic-infra/dynamic-agents-gcp.txt

@@ -103,10 +103,8 @@ an example configuration. See :ref:`cluster-configuration` for details.
      master_url: <scheme://host:port>
      startup_script: <startup script>
      container_startup_script: <container startup script>
-     agent_docker_network: host
-     agent_docker_image: <agent docker image>
+     agent_docker_network: determined
      max_idle_agent_period: 5m
-     max_agent_starting_period: 5m
 
      provider: gcp
      base_config: <instance resource base configuration>

master/internal/config_test.go

@@ -31,7 +31,6 @@ scheduler:
 
 provisioner:
   max_idle_agent_period: 30s
-  max_agent_starting_period: 30s
 `
 	expected := Config{
 		Log: logger.Config{
@@ -46,10 +45,9 @@ provisioner:
 		},
 		Scheduler: scheduler.Config{Fit: "best"},
 		Provisioner: &provisioner.Config{
-			AgentDockerRuntime:     "runc",
-			AgentDockerNetwork:     "default",
-			MaxIdleAgentPeriod:     provisioner.Duration(30 * time.Second),
-			MaxAgentStartingPeriod: provisioner.Duration(30 * time.Second),
+			AgentDockerRuntime: "runc",
+			AgentDockerNetwork: "default",
+			MaxIdleAgentPeriod: provisioner.Duration(30 * time.Second),
 		},
 	}
 

master/internal/provisioner/config.go

@@ -53,16 +53,14 @@ type Config struct {
 	AWS                    *AWSClusterConfig `union:"provider,aws" json:"-"`
 	GCP                    *GCPClusterConfig `union:"provider,gcp" json:"-"`
 	MaxIdleAgentPeriod     Duration          `json:"max_idle_agent_period"`
-	MaxAgentStartingPeriod Duration          `json:"max_agent_starting_period"`
 }
 
 // DefaultConfig returns the default configuration of the provisioner.
 func DefaultConfig() *Config {
 	return &Config{
-		AgentDockerRuntime:     "runc",
-		AgentDockerNetwork:     "default",
-		MaxIdleAgentPeriod:     Duration(300 * time.Second),
-		MaxAgentStartingPeriod: Duration(300 * time.Second),
+		AgentDockerRuntime: "runc",
+		AgentDockerNetwork: "default",
+		MaxIdleAgentPeriod: Duration(300 * time.Second),
 	}
 }
 
@@ -102,8 +100,6 @@ func (c Config) Validate() []error {
 		check.False(c.AWS == nil && c.GCP == nil, "must configure aws or gcp cluster"),
 		check.GreaterThan(
 			int64(c.MaxIdleAgentPeriod), int64(0), "max idle agent period must be greater than 0"),
-		check.GreaterThan(
-			int64(c.MaxAgentStartingPeriod), int64(0), "max agent starting period must be greater than 0"),
 	}...)
 	return errs
 }

master/internal/provisioner/config_test.go

@@ -18,10 +18,9 @@ func TestProvisionerConfigMissingFields(t *testing.T) {
 	err = check.Validate(&config)
 	assert.ErrorContains(t, err, "must configure aws or gcp cluster")
 	expected := Config{
-		MaxIdleAgentPeriod:     Duration(5 * time.Minute),
-		MaxAgentStartingPeriod: Duration(5 * time.Minute),
-		AgentDockerRuntime:     "runc",
-		AgentDockerNetwork:     "default",
+		MaxIdleAgentPeriod: Duration(5 * time.Minute),
+		AgentDockerRuntime: "runc",
+		AgentDockerNetwork: "default",
 	}
 	assert.DeepEqual(t, config, expected)
 }
@@ -34,8 +33,7 @@ func TestUnmarshalProvisionerConfigMasterURL(t *testing.T) {
 "region": "test.region3",
 "image_id": "test.image3",
 "ssh_key_name": "test-key3",
-"max_idle_agent_period": "30s",
-"max_agent_starting_period": "30s"
+"max_idle_agent_period": "30s"
 }`
 	config := Config{}
 	err := json.Unmarshal([]byte(configRaw), &config)
@@ -49,13 +47,12 @@ func TestUnmarshalProvisionerConfigMasterURL(t *testing.T) {
 	awsConfig.ImageID = "test.image3"
 	awsConfig.SSHKeyName = "test-key3"
 	unmarshaled := Config{
-		MasterURL:              "http://test.master:8080",
-		AgentDockerImage:       "test_image",
-		AgentDockerRuntime:     "runc",
-		AgentDockerNetwork:     "default",
-		AWS:                    &awsConfig,
-		MaxIdleAgentPeriod:     Duration(30 * time.Second),
-		MaxAgentStartingPeriod: Duration(30 * time.Second),
+		MasterURL:          "http://test.master:8080",
+		AgentDockerImage:   "test_image",
+		AgentDockerRuntime: "runc",
+		AgentDockerNetwork: "default",
+		AWS:                &awsConfig,
+		MaxIdleAgentPeriod: Duration(30 * time.Second),
 	}
 	assert.DeepEqual(t, config, unmarshaled)
 }
@@ -83,8 +80,7 @@ func TestUnmarshalProvisionerConfigWithAWS(t *testing.T) {
 "region": "test.region2",
 "image_id": "test.image2",
 "ssh_key_name": "test-key2",
-"max_idle_agent_period": "30s",
-"max_agent_starting_period": "30s"
+"max_idle_agent_period": "30s"
 }`
 	config := Config{}
 	err := json.Unmarshal([]byte(configRaw), &config)
@@ -98,13 +94,12 @@ func TestUnmarshalProvisionerConfigWithAWS(t *testing.T) {
 	awsConfig.ImageID = "test.image2"
 	awsConfig.SSHKeyName = "test-key2"
 	unmarshaled := Config{
-		MasterURL:              "http://test.master:8080",
-		AWS:                    &awsConfig,
-		AgentDockerImage:       "test_image",
-		AgentDockerRuntime:     "runc",
-		AgentDockerNetwork:     "default",
-		MaxIdleAgentPeriod:     Duration(30 * time.Second),
-		MaxAgentStartingPeriod: Duration(30 * time.Second),
+		MasterURL:          "http://test.master:8080",
+		AWS:                &awsConfig,
+		AgentDockerImage:   "test_image",
+		AgentDockerRuntime: "runc",
+		AgentDockerNetwork: "default",
+		MaxIdleAgentPeriod: Duration(30 * time.Second),
 	}
 	assert.DeepEqual(t, config, unmarshaled)
 }
@@ -130,13 +125,12 @@ func TestUnmarshalProvisionerConfigWithGCP(t *testing.T) {
 	expected.Zone = "test-zone2"
 	expected.BootDiskSourceImage = "test-source_image2"
 	unmarshaled := Config{
-		MasterURL:              "http://test.master:8080",
-		GCP:                    &expected,
-		AgentDockerImage:       "test_image",
-		AgentDockerRuntime:     "runc",
-		AgentDockerNetwork:     "default",
-		MaxIdleAgentPeriod:     Duration(5 * time.Minute),
-		MaxAgentStartingPeriod: Duration(5 * time.Minute),
+		MasterURL:          "http://test.master:8080",
+		GCP:                &expected,
+		AgentDockerImage:   "test_image",
+		AgentDockerRuntime: "runc",
+		AgentDockerNetwork: "default",
+		MaxIdleAgentPeriod: Duration(5 * time.Minute),
 	}
 	assert.DeepEqual(t, config, unmarshaled)
 }

master/internal/provisioner/provisioner.go

@@ -62,11 +62,8 @@ func New(config *Config) (*Provisioner, error) {
 	}
 
 	return &Provisioner{
-		provider: cluster,
-		scaleDecider: newScaleDecider(
-			time.Duration(config.MaxIdleAgentPeriod),
-			time.Duration(config.MaxAgentStartingPeriod),
-		),
+		provider:     cluster,
+		scaleDecider: newScaleDecider(time.Duration(config.MaxIdleAgentPeriod)),
 	}, nil
 }