Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] azurerm_hdinsight_hadoop_cluster - Add edge node support #4049

Closed
wants to merge 13 commits into from
Closed

[WIP] azurerm_hdinsight_hadoop_cluster - Add edge node support #4049

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
b4767c1
Start of edge node addition
mbfrahry Aug 9, 2019
f59db70
Add application type
mbfrahry Aug 9, 2019
5a56e15
Remove abstraction
mbfrahry Aug 9, 2019
cc272cf
Fmt
mbfrahry Aug 9, 2019
fe4cb79
Fix test
mbfrahry Aug 9, 2019
b98a7bd
Add target instance count
mbfrahry Aug 9, 2019
9368566
Add install script
mbfrahry Aug 9, 2019
1b94e3a
Install scripts
mbfrahry Aug 13, 2019
8baacbf
Fmt
mbfrahry Aug 13, 2019
5cff846
s ->
mbfrahry Aug 13, 2019
5adb47b
Test help
mbfrahry Aug 13, 2019
a60ef10
RC1
mbfrahry Aug 21, 2019
3abfe64
Broken tests
mbfrahry Aug 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 67 additions & 2 deletions azurerm/common_hdinsight.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package azurerm

import (
"context"
"fmt"
"log"

"github.com/Azure/azure-sdk-for-go/services/preview/hdinsight/mgmt/2018-06-01-preview/hdinsight"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/helper/schema"
"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
"github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils"
"log"
"time"
)

func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema.UpdateFunc {
Expand Down Expand Up @@ -54,6 +56,38 @@ func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema
}
}

// The API can add an edge node but can't remove them without force newing the resource. We'll check for adding here
// and can come back to removing if that functionality gets added. https://feedback.azure.com/forums/217335-hdinsight/suggestions/5663773-start-stop-cluster-hdinsight?page=3&per_page=20
if clusterKind == "Hadoop" {
if d.HasChange("roles.0.edge_node") {
o, n := d.GetChange("roles.0.edge_node.#")
edgeNodeRaw := d.Get("roles.0.edge_node").([]interface{})
edgeNodeConfig := edgeNodeRaw[0].(map[string]interface{})
applicationsClient := meta.(*ArmClient).hdinsight.ApplicationsClient

// Create an edge node
if o.(int) < n.(int) {
err := createHDInsightEdgeNode(applicationsClient, ctx, resourceGroup, name, edgeNodeConfig)
if err != nil {
return err
}

// we can't rely on the use of the Future here due to the node being successfully completed but now the cluster is applying those changes.
log.Printf("[DEBUG] Waiting for Hadoop Cluster to %q (Resource Group %q) to finish applying edge node", name, resourceGroup)
stateConf := &resource.StateChangeConf{
Pending: []string{"AzureVMConfiguration", "Accepted", "HdInsightConfiguration"},
Target: []string{"Running"},
Refresh: hdInsightWaitForReadyRefreshFunc(ctx, client, resourceGroup, name),
Timeout: 60 * time.Minute,
MinTimeout: 15 * time.Second,
}
if _, err := stateConf.WaitForState(); err != nil {
return fmt.Errorf("Error waiting for HDInsight Cluster %q (Resource Group %q) to be running: %s", name, resourceGroup, err)
}
}
}
}

return readFunc(d, meta)
}
}
Expand Down Expand Up @@ -175,3 +209,34 @@ func flattenHDInsightRoles(d *schema.ResourceData, input *hdinsight.ComputeProfi
result,
}
}

func createHDInsightEdgeNode(client hdinsight.ApplicationsClient, ctx context.Context, resourceGroup string, name string, input map[string]interface{}) error {
installScriptActions := expandHDInsightApplicationEdgeNodeInstallScriptActions(input["install_script_action"].([]interface{}))

application := hdinsight.Application{
Properties: &hdinsight.ApplicationProperties{
ComputeProfile: &hdinsight.ComputeProfile{
Roles: &[]hdinsight.Role{{
Name: utils.String("edgenode"),
HardwareProfile: &hdinsight.HardwareProfile{
VMSize: utils.String(input["vm_size"].(string)),
},
// The TargetInstanceCount must be one for edge nodes.
TargetInstanceCount: utils.Int32(1),
}},
},
InstallScriptActions: installScriptActions,
ApplicationType: utils.String("CustomApplication"),
},
}
future, err := client.Create(ctx, resourceGroup, name, name, application)
if err != nil {
return fmt.Errorf("Error creating edge node for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
return fmt.Errorf("Error waiting for creation of edge node for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

return nil
}
182 changes: 94 additions & 88 deletions azurerm/helpers/azure/hdinsight.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/hashicorp/go-getter/helper/url"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/helper/validation"
"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/suppress"
"github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/validate"
"github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils"
)
Expand Down Expand Up @@ -212,97 +213,102 @@ type HDInsightNodeDefinition struct {
FixedTargetInstanceCount *int32
}

func ValidateSchemaHDInsightNodeDefinitionVMSize() schema.SchemaValidateFunc {
return validation.StringInSlice([]string{
// short of deploying every VM Sku for every node type for every HDInsight Cluster
// this is the list I've (@tombuildsstuff) found for valid SKU's from an endpoint in the Portal
// using another SKU causes a bad request from the API - as such this is a best effort UX
"ExtraSmall",
"Small",
"Medium",
"Large",
"ExtraLarge",
"A5",
"A6",
"A7",
"A8",
"A9",
"A10",
"A11",
"Standard_A1_V2",
"Standard_A2_V2",
"Standard_A2m_V2",
"Standard_A3",
"Standard_A4_V2",
"Standard_A4m_V2",
"Standard_A8_V2",
"Standard_A8m_V2",
"Standard_D1",
"Standard_D2",
"Standard_D3",
"Standard_D4",
"Standard_D11",
"Standard_D12",
"Standard_D13",
"Standard_D14",
"Standard_D1_V2",
"Standard_D2_V2",
"Standard_D3_V2",
"Standard_D4_V2",
"Standard_D5_V2",
"Standard_D11_V2",
"Standard_D12_V2",
"Standard_D13_V2",
"Standard_D14_V2",
"Standard_DS1_V2",
"Standard_DS2_V2",
"Standard_DS3_V2",
"Standard_DS4_V2",
"Standard_DS5_V2",
"Standard_DS11_V2",
"Standard_DS12_V2",
"Standard_DS13_V2",
"Standard_DS14_V2",
"Standard_E2_V3",
"Standard_E4_V3",
"Standard_E8_V3",
"Standard_E16_V3",
"Standard_E20_V3",
"Standard_E32_V3",
"Standard_E64_V3",
"Standard_E64i_V3",
"Standard_E2s_V3",
"Standard_E4s_V3",
"Standard_E8s_V3",
"Standard_E16s_V3",
"Standard_E20s_V3",
"Standard_E32s_V3",
"Standard_E64s_V3",
"Standard_E64is_V3",
"Standard_G1",
"Standard_G2",
"Standard_G3",
"Standard_G4",
"Standard_G5",
"Standard_F2s_V2",
"Standard_F4s_V2",
"Standard_F8s_V2",
"Standard_F16s_V2",
"Standard_F32s_V2",
"Standard_F64s_V2",
"Standard_F72s_V2",
"Standard_GS1",
"Standard_GS2",
"Standard_GS3",
"Standard_GS4",
"Standard_GS5",
"Standard_NC24",
}, true)
}

func SchemaHDInsightNodeDefinition(schemaLocation string, definition HDInsightNodeDefinition) *schema.Schema {
result := map[string]*schema.Schema{
"vm_size": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{
// short of deploying every VM Sku for every node type for every HDInsight Cluster
// this is the list I've (@tombuildsstuff) found for valid SKU's from an endpoint in the Portal
// using another SKU causes a bad request from the API - as such this is a best effort UX
"ExtraSmall",
"Small",
"Medium",
"Large",
"ExtraLarge",
"A5",
"A6",
"A7",
"A8",
"A9",
"A10",
"A11",
"Standard_A1_V2",
"Standard_A2_V2",
"Standard_A2m_V2",
"Standard_A3",
"Standard_A4_V2",
"Standard_A4m_V2",
"Standard_A8_V2",
"Standard_A8m_V2",
"Standard_D1",
"Standard_D2",
"Standard_D3",
"Standard_D4",
"Standard_D11",
"Standard_D12",
"Standard_D13",
"Standard_D14",
"Standard_D1_V2",
"Standard_D2_V2",
"Standard_D3_V2",
"Standard_D4_V2",
"Standard_D5_V2",
"Standard_D11_V2",
"Standard_D12_V2",
"Standard_D13_V2",
"Standard_D14_V2",
"Standard_DS1_V2",
"Standard_DS2_V2",
"Standard_DS3_V2",
"Standard_DS4_V2",
"Standard_DS5_V2",
"Standard_DS11_V2",
"Standard_DS12_V2",
"Standard_DS13_V2",
"Standard_DS14_V2",
"Standard_E2_V3",
"Standard_E4_V3",
"Standard_E8_V3",
"Standard_E16_V3",
"Standard_E20_V3",
"Standard_E32_V3",
"Standard_E64_V3",
"Standard_E64i_V3",
"Standard_E2s_V3",
"Standard_E4s_V3",
"Standard_E8s_V3",
"Standard_E16s_V3",
"Standard_E20s_V3",
"Standard_E32s_V3",
"Standard_E64s_V3",
"Standard_E64is_V3",
"Standard_G1",
"Standard_G2",
"Standard_G3",
"Standard_G4",
"Standard_G5",
"Standard_F2s_V2",
"Standard_F4s_V2",
"Standard_F8s_V2",
"Standard_F16s_V2",
"Standard_F32s_V2",
"Standard_F64s_V2",
"Standard_F72s_V2",
"Standard_GS1",
"Standard_GS2",
"Standard_GS3",
"Standard_GS4",
"Standard_GS5",
"Standard_NC24",
}, true),
Type: schema.TypeString,
Required: true,
ForceNew: true,
DiffSuppressFunc: suppress.CaseDifference,
ValidateFunc: ValidateSchemaHDInsightNodeDefinitionVMSize(),
},
"username": {
Type: schema.TypeString,
Expand Down
Loading