diff --git a/aws/resource_aws_glue_job.go b/aws/resource_aws_glue_job.go index 3a296c393b7a..2e6415c3e299 100644 --- a/aws/resource_aws_glue_job.go +++ b/aws/resource_aws_glue_job.go @@ -24,7 +24,8 @@ func resourceAwsGlueJob() *schema.Resource { "allocated_capacity": { Type: schema.TypeInt, Optional: true, - Default: 10, + Computed: true, + Deprecated: "Please use attribute `max_capacity' instead. This attribute might be removed in future releases.", ValidateFunc: validation.IntAtLeast(2), }, "command": { @@ -74,6 +75,11 @@ func resourceAwsGlueJob() *schema.Resource { }, }, }, + "max_capacity": { + Type: schema.TypeFloat, + Optional: true, + Computed: true, + }, "max_retries": { Type: schema.TypeInt, Optional: true, @@ -114,8 +120,13 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error { Timeout: aws.Int64(int64(d.Get("timeout").(int))), } - if v, ok := d.GetOk("allocated_capacity"); ok { - input.AllocatedCapacity = aws.Int64(int64(v.(int))) + if v, ok := d.GetOk("max_capacity"); ok { + input.MaxCapacity = aws.Float64(v.(float64)) + } else { + if v, ok := d.GetOk("allocated_capacity"); ok { + input.MaxCapacity = aws.Float64(float64(v.(int))) + log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.") + } } if v, ok := d.GetOk("connections"); ok { @@ -184,7 +195,6 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error { return nil } - d.Set("allocated_capacity", int(aws.Int64Value(job.AllocatedCapacity))) if err := d.Set("command", flattenGlueJobCommand(job.Command)); err != nil { return fmt.Errorf("error setting command: %s", err) } @@ -198,6 +208,7 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error { if err := d.Set("execution_property", flattenGlueExecutionProperty(job.ExecutionProperty)); err != nil { return fmt.Errorf("error setting execution_property: %s", err) } + d.Set("max_capacity", aws.Float64Value(job.MaxCapacity)) d.Set("max_retries", int(aws.Int64Value(job.MaxRetries))) d.Set("name", job.Name) d.Set("role_arn", job.Role) @@ -206,6 +217,9 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error { return fmt.Errorf("error setting security_configuration: %s", err) } + // TODO: Deprecated fields - remove in next major version + d.Set("allocated_capacity", int(aws.Int64Value(job.AllocatedCapacity))) + return nil } @@ -218,8 +232,13 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { Timeout: aws.Int64(int64(d.Get("timeout").(int))), } - if v, ok := d.GetOk("allocated_capacity"); ok { - jobUpdate.AllocatedCapacity = aws.Int64(int64(v.(int))) + if v, ok := d.GetOk("max_capacity"); ok { + jobUpdate.MaxCapacity = aws.Float64(v.(float64)) + } + + if d.HasChange("allocated_capacity") { + jobUpdate.MaxCapacity = aws.Float64(float64(d.Get("allocated_capacity").(int))) + log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.") } if v, ok := d.GetOk("connections"); ok { @@ -263,7 +282,7 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err) } - return nil + return resourceAwsGlueJobRead(d, meta) } func resourceAwsGlueJobDelete(d *schema.ResourceData, meta interface{}) error { diff --git a/aws/resource_aws_glue_job_test.go b/aws/resource_aws_glue_job_test.go index aa12b0c6f7ec..dd8556227255 100644 --- a/aws/resource_aws_glue_job_test.go +++ b/aws/resource_aws_glue_job_test.go @@ -395,6 +395,73 @@ func TestAccAWSGlueJob_SecurityConfiguration(t *testing.T) { }) } +func TestAccAWSGlueJob_PythonShell(t *testing.T) { + var job glue.Job + + rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5)) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSGlueJobConfig_PythonShell(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "max_capacity", "0.0625"), + resource.TestCheckResourceAttr(resourceName, "command.#", "1"), + resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"), + resource.TestCheckResourceAttr(resourceName, "command.0.name", "pythonshell"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccAWSGlueJob_MaxCapacity(t *testing.T) { + var job glue.Job + + rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5)) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSGlueJobConfig_MaxCapacity(rName, 10), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "max_capacity", "10"), + resource.TestCheckResourceAttr(resourceName, "command.#", "1"), + resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"), + resource.TestCheckResourceAttr(resourceName, "command.0.name", "glueetl"), + ), + }, + { + Config: testAccAWSGlueJobConfig_MaxCapacity(rName, 15), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "max_capacity", "15"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func testAccCheckAWSGlueJobExists(resourceName string, job *glue.Job) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[resourceName] @@ -516,6 +583,7 @@ func testAccAWSGlueJobConfig_Command(rName, scriptLocation string) string { resource "aws_glue_job" "test" { name = "%s" role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 command { script_location = "%s" @@ -533,6 +601,7 @@ func testAccAWSGlueJobConfig_DefaultArguments(rName, jobBookmarkOption, jobLangu resource "aws_glue_job" "test" { name = "%s" role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -556,6 +625,7 @@ resource "aws_glue_job" "test" { description = "%s" name = "%s" role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -573,6 +643,7 @@ func testAccAWSGlueJobConfig_ExecutionProperty(rName string, maxConcurrentRuns i resource "aws_glue_job" "test" { name = "%s" role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -595,6 +666,7 @@ resource "aws_glue_job" "test" { max_retries = %d name = "%s" role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -612,6 +684,7 @@ func testAccAWSGlueJobConfig_Required(rName string) string { resource "aws_glue_job" "test" { name = "%s" role_arn = "${aws_iam_role.test.arn}" + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -630,6 +703,7 @@ resource "aws_glue_job" "test" { name = "%s" role_arn = "${aws_iam_role.test.arn}" timeout = %d + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -648,6 +722,7 @@ resource "aws_glue_job" "test" { name = "%s" role_arn = "${aws_iam_role.test.arn}" security_configuration = "%s" + allocated_capacity = 10 command { script_location = "testscriptlocation" @@ -657,3 +732,40 @@ resource "aws_glue_job" "test" { } `, testAccAWSGlueJobConfig_Base(rName), rName, securityConfiguration) } + +func testAccAWSGlueJobConfig_PythonShell(rName string) string { + return fmt.Sprintf(` +%s + +resource "aws_glue_job" "test" { + name = "%s" + role_arn = "${aws_iam_role.test.arn}" + max_capacity = 0.0625 + + command { + name="pythonshell" + script_location = "testscriptlocation" + } + + depends_on = ["aws_iam_role_policy_attachment.test"] +} +`, testAccAWSGlueJobConfig_Base(rName), rName) +} + +func testAccAWSGlueJobConfig_MaxCapacity(rName string, maxCapacity float64) string { + return fmt.Sprintf(` +%s + +resource "aws_glue_job" "test" { + name = "%s" + role_arn = "${aws_iam_role.test.arn}" + max_capacity = %g + + command { + script_location = "testscriptlocation" + } + + depends_on = ["aws_iam_role_policy_attachment.test"] +} +`, testAccAWSGlueJobConfig_Base(rName), rName, maxCapacity) +} diff --git a/website/docs/r/glue_job.html.markdown b/website/docs/r/glue_job.html.markdown index 1e280de8bd2c..1af75cf56946 100644 --- a/website/docs/r/glue_job.html.markdown +++ b/website/docs/r/glue_job.html.markdown @@ -46,12 +46,16 @@ resource "aws_glue_job" "example" { The following arguments are supported: -* `allocated_capacity` – (Optional) The number of AWS Glue data processing units (DPUs) to allocate to this Job. At least 2 DPUs need to be allocated; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. +~> **NOTE:** The `allocated_capacity` attribute has been deprecated and might +be removed in future releases, please use `max_capacity` instead. + +* `allocated_capacity` – **DEPRECATED** (Optional) The number of AWS Glue data processing units (DPUs) to allocate to this Job. At least 2 DPUs need to be allocated; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. * `command` – (Required) The command of the job. Defined below. * `connections` – (Optional) The list of connections used for this job. * `default_arguments` – (Optional) The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide. * `description` – (Optional) Description of the job. * `execution_property` – (Optional) Execution property of the job. Defined below. +* `max_capacity` – (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. * `max_retries` – (Optional) The maximum number of times to retry this job if it fails. * `name` – (Required) The name you assign to this job. It must be unique in your account. * `role_arn` – (Required) The ARN of the IAM role associated with this job.