Skip to content

Commit

Permalink
Fix aws_glue_job resource attribute
Browse files Browse the repository at this point in the history
  • Loading branch information
@teraken0509
teraken0509 committed Jan 26, 2019
1 parent 5b8ed28 commit 0c47022
Show file tree
Hide file tree
Showing 3 changed files with 143 additions and 8 deletions.
33 changes: 26 additions & 7 deletions aws/resource_aws_glue_job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ func resourceAwsGlueJob() *schema.Resource {
"allocated_capacity": {
Type: schema.TypeInt,
Optional: true,
Default: 10,
Computed: true,
Deprecated: "Please use attribute `max_capacity' instead. This attribute might be removed in future releases.",
ValidateFunc: validation.IntAtLeast(2),
},
"command": {
Expand Down Expand Up @@ -74,6 +75,11 @@ func resourceAwsGlueJob() *schema.Resource {
},
},
},
"max_capacity": {
Type: schema.TypeFloat,
Optional: true,
Computed: true,
},
"max_retries": {
Type: schema.TypeInt,
Optional: true,
Expand Down Expand Up @@ -114,8 +120,13 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error {
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}

if v, ok := d.GetOk("allocated_capacity"); ok {
input.AllocatedCapacity = aws.Int64(int64(v.(int)))
if v, ok := d.GetOk("max_capacity"); ok {
input.MaxCapacity = aws.Float64(v.(float64))
} else {
if v, ok := d.GetOk("allocated_capacity"); ok {
input.MaxCapacity = aws.Float64(float64(v.(int)))
log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.")
}
}

if v, ok := d.GetOk("connections"); ok {
Expand Down Expand Up @@ -184,7 +195,6 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
return nil
}

d.Set("allocated_capacity", int(aws.Int64Value(job.AllocatedCapacity)))
if err := d.Set("command", flattenGlueJobCommand(job.Command)); err != nil {
return fmt.Errorf("error setting command: %s", err)
}
Expand All @@ -198,6 +208,7 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
if err := d.Set("execution_property", flattenGlueExecutionProperty(job.ExecutionProperty)); err != nil {
return fmt.Errorf("error setting execution_property: %s", err)
}
d.Set("max_capacity", aws.Float64Value(job.MaxCapacity))
d.Set("max_retries", int(aws.Int64Value(job.MaxRetries)))
d.Set("name", job.Name)
d.Set("role_arn", job.Role)
Expand All @@ -206,6 +217,9 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
return fmt.Errorf("error setting security_configuration: %s", err)
}

// TODO: Deprecated fields - remove in next major version
d.Set("allocated_capacity", int(aws.Int64Value(job.AllocatedCapacity)))

return nil
}

Expand All @@ -218,8 +232,13 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error {
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}

if v, ok := d.GetOk("allocated_capacity"); ok {
jobUpdate.AllocatedCapacity = aws.Int64(int64(v.(int)))
if v, ok := d.GetOk("max_capacity"); ok {
jobUpdate.MaxCapacity = aws.Float64(v.(float64))
}

if d.HasChange("allocated_capacity") {
jobUpdate.MaxCapacity = aws.Float64(float64(d.Get("allocated_capacity").(int)))
log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.")
}

if v, ok := d.GetOk("connections"); ok {
Expand Down Expand Up @@ -263,7 +282,7 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error {
return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err)
}

return nil
return resourceAwsGlueJobRead(d, meta)
}

func resourceAwsGlueJobDelete(d *schema.ResourceData, meta interface{}) error {
Expand Down
112 changes: 112 additions & 0 deletions aws/resource_aws_glue_job_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,73 @@ func TestAccAWSGlueJob_SecurityConfiguration(t *testing.T) {
})
}

func TestAccAWSGlueJob_PythonShell(t *testing.T) {
var job glue.Job

rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
resourceName := "aws_glue_job.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueJobDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSGlueJobConfig_PythonShell(rName),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job),
resource.TestCheckResourceAttr(resourceName, "max_capacity", "0.0625"),
resource.TestCheckResourceAttr(resourceName, "command.#", "1"),
resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"),
resource.TestCheckResourceAttr(resourceName, "command.0.name", "pythonshell"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func TestAccAWSGlueJob_MaxCapacity(t *testing.T) {
var job glue.Job

rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
resourceName := "aws_glue_job.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueJobDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSGlueJobConfig_MaxCapacity(rName, 10),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job),
resource.TestCheckResourceAttr(resourceName, "max_capacity", "10"),
resource.TestCheckResourceAttr(resourceName, "command.#", "1"),
resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"),
resource.TestCheckResourceAttr(resourceName, "command.0.name", "glueetl"),
),
},
{
Config: testAccAWSGlueJobConfig_MaxCapacity(rName, 15),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job),
resource.TestCheckResourceAttr(resourceName, "max_capacity", "15"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func testAccCheckAWSGlueJobExists(resourceName string, job *glue.Job) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[resourceName]
Expand Down Expand Up @@ -516,6 +583,7 @@ func testAccAWSGlueJobConfig_Command(rName, scriptLocation string) string {
resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
allocated_capacity = 10
command {
script_location = "%s"
Expand All @@ -533,6 +601,7 @@ func testAccAWSGlueJobConfig_DefaultArguments(rName, jobBookmarkOption, jobLangu
resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -556,6 +625,7 @@ resource "aws_glue_job" "test" {
description = "%s"
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -573,6 +643,7 @@ func testAccAWSGlueJobConfig_ExecutionProperty(rName string, maxConcurrentRuns i
resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -595,6 +666,7 @@ resource "aws_glue_job" "test" {
max_retries = %d
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -612,6 +684,7 @@ func testAccAWSGlueJobConfig_Required(rName string) string {
resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -630,6 +703,7 @@ resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
timeout = %d
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -648,6 +722,7 @@ resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
security_configuration = "%s"
allocated_capacity = 10
command {
script_location = "testscriptlocation"
Expand All @@ -657,3 +732,40 @@ resource "aws_glue_job" "test" {
}
`, testAccAWSGlueJobConfig_Base(rName), rName, securityConfiguration)
}

func testAccAWSGlueJobConfig_PythonShell(rName string) string {
return fmt.Sprintf(`
%s
resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
max_capacity = 0.0625
command {
name="pythonshell"
script_location = "testscriptlocation"
}
depends_on = ["aws_iam_role_policy_attachment.test"]
}
`, testAccAWSGlueJobConfig_Base(rName), rName)
}

func testAccAWSGlueJobConfig_MaxCapacity(rName string, maxCapacity float64) string {
return fmt.Sprintf(`
%s
resource "aws_glue_job" "test" {
name = "%s"
role_arn = "${aws_iam_role.test.arn}"
max_capacity = %g
command {
script_location = "testscriptlocation"
}
depends_on = ["aws_iam_role_policy_attachment.test"]
}
`, testAccAWSGlueJobConfig_Base(rName), rName, maxCapacity)
}
6 changes: 5 additions & 1 deletion website/docs/r/glue_job.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,16 @@ resource "aws_glue_job" "example" {

The following arguments are supported:

* `allocated_capacity` – (Optional) The number of AWS Glue data processing units (DPUs) to allocate to this Job. At least 2 DPUs need to be allocated; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory.
~> **NOTE:** The `allocated_capacity` attribute has been deprecated and might
be removed in future releases, please use `max_capacity` instead.

* `allocated_capacity` – **DEPRECATED** (Optional) The number of AWS Glue data processing units (DPUs) to allocate to this Job. At least 2 DPUs need to be allocated; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory.
* `command` – (Required) The command of the job. Defined below.
* `connections` – (Optional) The list of connections used for this job.
* `default_arguments` – (Optional) The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide.
* `description` – (Optional) Description of the job.
* `execution_property` – (Optional) Execution property of the job. Defined below.
* `max_capacity` – (Optional) The maximum number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.
* `max_retries` – (Optional) The maximum number of times to retry this job if it fails.
* `name` – (Required) The name you assign to this job. It must be unique in your account.
* `role_arn` – (Required) The ARN of the IAM role associated with this job.
Expand Down

0 comments on commit 0c47022

Please sign in to comment.