Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Azure monitor support for azurerm_hdinsight_*_cluster #7045

Merged
merged 51 commits into from
Jul 9, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
8f6e1c8
Enable monitor on create
kosinsky May 19, 2020
6497a82
Implement monitor changes
kosinsky May 19, 2020
20ca28b
Error message clean up
kosinsky May 19, 2020
6b8ac0d
Documentation for hadoop
kosinsky May 19, 2020
af64be0
A bit of refactoring
kosinsky May 19, 2020
e45e69a
Aze monioor HBase cluster
kosinsky May 19, 2020
5566f35
Azure Monitor for InteractiveQuery cluster
kosinsky May 19, 2020
1c55195
Azure monitor for Kafka
kosinsky May 19, 2020
475d205
Azure monitor for Spark
kosinsky May 19, 2020
416cbb2
Azure monitor for Storm
kosinsky May 20, 2020
1220e10
Azure monitor doc for all hdinsight resources
kosinsky May 20, 2020
abdd57a
Ensure that basic and monitor tests use the same roles
kosinsky May 21, 2020
1a16f34
Fixing linter errors
kosinsky May 21, 2020
6c79c3f
Merge branch 'master' into hdi_azure_monitor
kosinsky Jun 30, 2020
1bff630
Fix typo in debug output
kosinsky Jun 30, 2020
07b7624
Moved extensionsClient
kosinsky Jul 1, 2020
1a47098
Return error if disabling monitor fails
kosinsky Jul 1, 2020
1c451ff
Error messages
kosinsky Jul 7, 2020
8227f7c
Error messages
kosinsky Jul 7, 2020
cf39c28
Error messages
kosinsky Jul 7, 2020
e2a7c85
Error messages
kosinsky Jul 7, 2020
7dc5092
Error messages
kosinsky Jul 7, 2020
186a09c
Error messages
kosinsky Jul 7, 2020
b2365c1
Remove redundant ForceNew
kosinsky Jul 7, 2020
6738651
Enable monitor on create
kosinsky May 19, 2020
930b73b
Implement monitor changes
kosinsky May 19, 2020
42d8cf0
Error message clean up
kosinsky May 19, 2020
85824a6
Documentation for hadoop
kosinsky May 19, 2020
21e9d99
A bit of refactoring
kosinsky May 19, 2020
c9df321
Aze monioor HBase cluster
kosinsky May 19, 2020
b5cf744
Azure Monitor for InteractiveQuery cluster
kosinsky May 19, 2020
5fca7ae
Azure monitor for Kafka
kosinsky May 19, 2020
b1609cd
Azure monitor for Spark
kosinsky May 19, 2020
36f4e18
Azure monitor for Storm
kosinsky May 20, 2020
972c1b7
Azure monitor doc for all hdinsight resources
kosinsky May 20, 2020
da5845f
Ensure that basic and monitor tests use the same roles
kosinsky May 21, 2020
0b96308
Fixing linter errors
kosinsky May 21, 2020
dfa757a
Fix typo in debug output
kosinsky Jun 30, 2020
f5308cd
Moved extensionsClient
kosinsky Jul 1, 2020
0b89eb5
Return error if disabling monitor fails
kosinsky Jul 1, 2020
7350c63
Error messages
kosinsky Jul 7, 2020
de60a55
Error messages
kosinsky Jul 7, 2020
ef9b7a0
Error messages
kosinsky Jul 7, 2020
f9f25c6
Error messages
kosinsky Jul 7, 2020
033cb71
Error messages
kosinsky Jul 7, 2020
31c6188
post conflict rebase cleanup and catchup for hdinsight service
jackofallops Jul 8, 2020
2abd334
whitespace fixes
jackofallops Jul 8, 2020
e3d0f38
Merge branch 'hdi_azure_monitor' of github.com:kosinsky/terraform-pro…
kosinsky Jul 8, 2020
1902856
Validation for log_analytics_workspace_id
kosinsky Jul 8, 2020
bae25a4
Merge branch 'hdi_azure_monitor' of github.com:kosinsky/terraform-pro…
kosinsky Jul 8, 2020
6745f26
Fix bad merbge and move extensionsClient
kosinsky Jul 8, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions azurerm/helpers/azure/hdinsight.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,33 @@ func SchemaHDInsightsExternalMetastores() *schema.Schema {
}
}

func SchemaHDInsightsMonitor() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"log_analytics_workspace_id": {
Type: schema.TypeString,
Required: true,
ValidateFunc: validation.IsUUID,
},
"primary_key": {
Type: schema.TypeString,
Required: true,
Sensitive: true,
ValidateFunc: validation.StringIsNotEmpty,
// Azure doesn't return the key
DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
return (new == d.Get(k).(string)) && (old == "*****")
},
},
},
},
}
}

func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

Expand Down Expand Up @@ -278,6 +305,18 @@ func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{}
}
}

func ExpandHDInsightsMonitor(input []interface{}) hdinsight.ClusterMonitoringRequest {
vs := input[0].(map[string]interface{})

workspace := vs["log_analytics_workspace_id"].(string)
key := vs["primary_key"].(string)

return hdinsight.ClusterMonitoringRequest{
WorkspaceID: utils.String(workspace),
PrimaryKey: utils.String(key),
}
}

func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
enabled := true

Expand Down
5 changes: 5 additions & 0 deletions azurerm/internal/services/hdinsight/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ type Client struct {
ApplicationsClient *hdinsight.ApplicationsClient
ClustersClient *hdinsight.ClustersClient
ConfigurationsClient *hdinsight.ConfigurationsClient
ExtensionsClient *hdinsight.ExtensionsClient
}

func NewClient(o *common.ClientOptions) *Client {
Expand All @@ -21,9 +22,13 @@ func NewClient(o *common.ClientOptions) *Client {
ConfigurationsClient := hdinsight.NewConfigurationsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
o.ConfigureClient(&ConfigurationsClient.Client, o.ResourceManagerAuthorizer)

ExtensionsClient := hdinsight.NewExtensionsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
o.ConfigureClient(&ExtensionsClient.Client, o.ResourceManagerAuthorizer)

return &Client{
ApplicationsClient: &ApplicationsClient,
ClustersClient: &ClustersClient,
ConfigurationsClient: &ConfigurationsClient,
ExtensionsClient: &ExtensionsClient,
}
}
51 changes: 51 additions & 0 deletions azurerm/internal/services/hdinsight/common_hdinsight.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema.UpdateFunc {
return func(d *schema.ResourceData, meta interface{}) error {
client := meta.(*clients.Client).HDInsight.ClustersClient
extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d)
defer cancel()

Expand Down Expand Up @@ -106,6 +107,17 @@ func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema
}
}

if d.HasChange("monitor") {
log.Printf("[DEBUG] Change Azure Monitor for the HDInsight %q Cluster", clusterKind)
if v, ok := d.GetOk("monitor"); ok {
monitorRaw := v.([]interface{})
if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
return err
}
} else if err := disableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name); err != nil {
return err
}
}
if d.HasChange("gateway") {
log.Printf("[DEBUG] Updating the HDInsight %q Cluster gateway", clusterKind)
vs := d.Get("gateway").([]interface{})[0].(map[string]interface{})
Expand Down Expand Up @@ -351,3 +363,42 @@ func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[stri
})
}
}

func flattenHDInsightMonitoring(monitor hdinsight.ClusterMonitoringResponse) []interface{} {
if *monitor.ClusterMonitoringEnabled {
return []interface{}{
map[string]string{
"log_analytics_workspace_id": *monitor.WorkspaceID,
"primary_key": "*****",
}}
}

return nil
}

func enableHDInsightMonitoring(ctx context.Context, client *hdinsight.ExtensionsClient, resourceGroup, name string, input []interface{}) error {
monitor := azure.ExpandHDInsightsMonitor(input)
future, err := client.EnableMonitoring(ctx, resourceGroup, name, monitor)
if err != nil {
return err
}

if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
return fmt.Errorf("Error waiting for enabling monitor for HDInsight Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

return nil
}

func disableHDInsightMonitoring(ctx context.Context, client *hdinsight.ExtensionsClient, resourceGroup, name string) error {
future, err := client.DisableMonitoring(ctx, resourceGroup, name)
if err != nil {
return err
}

if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
return fmt.Errorf("Error waiting for disabling monitor for HDInsight Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -166,12 +166,15 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {
Type: schema.TypeString,
Computed: true,
},

"monitor": azure.SchemaHDInsightsMonitor(),
},
}
}

func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interface{}) error {
client := meta.(*clients.Client).HDInsight.ClustersClient
extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
defer cancel()

Expand Down Expand Up @@ -199,7 +202,7 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, storageAccountsGen2Raw)
if err != nil {
return fmt.Errorf("Error expanding `storage_account`: %s", err)
return fmt.Errorf("failure expanding `storage_account`: %s", err)
}

rolesRaw := d.Get("roles").([]interface{})
Expand All @@ -210,14 +213,14 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
}
roles, err := expandHDInsightRoles(rolesRaw, hadoopRoles)
if err != nil {
return fmt.Errorf("Error expanding `roles`: %+v", err)
return fmt.Errorf("failure expanding `roles`: %+v", err)
}

if features.ShouldResourcesBeImported() {
existing, err := client.Get(ctx, resourceGroup, name)
if err != nil {
if !utils.ResponseWasNotFound(existing.Response) {
return fmt.Errorf("Error checking for presence of existing HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure checking for presence of existing HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}
}

Expand Down Expand Up @@ -250,20 +253,20 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
}
future, err := client.Create(ctx, resourceGroup, name, params)
if err != nil {
return fmt.Errorf("Error creating HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure creating HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
return fmt.Errorf("Error waiting for creation of HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failed waiting for creation of HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

read, err := client.Get(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

if read.ID == nil {
return fmt.Errorf("Error reading ID for HDInsight Hadoop Cluster %q (Resource Group %q)", name, resourceGroup)
return fmt.Errorf("failure reading ID for HDInsight Hadoop Cluster %q (Resource Group %q)", name, resourceGroup)
}

d.SetId(*read.ID)
Expand All @@ -290,7 +293,15 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
}

if _, err := stateConf.WaitForState(); err != nil {
return fmt.Errorf("Error waiting for HDInsight Cluster %q (Resource Group %q) to be running: %s", name, resourceGroup, err)
return fmt.Errorf("failure waiting for HDInsight Cluster %q (Resource Group %q) to be running: %s", name, resourceGroup, err)
}
}

// We can only enable monitoring after creation
if v, ok := d.GetOk("monitor"); ok {
monitorRaw := v.([]interface{})
if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
return err
}
}

Expand All @@ -300,6 +311,7 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interface{}) error {
clustersClient := meta.(*clients.Client).HDInsight.ClustersClient
configurationsClient := meta.(*clients.Client).HDInsight.ConfigurationsClient
extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d)
defer cancel()

Expand All @@ -319,18 +331,18 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
return nil
}

return fmt.Errorf("Error retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
Expand All @@ -347,11 +359,11 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac

if def := props.ClusterDefinition; def != nil {
if err := d.Set("component_version", flattenHDInsightHadoopComponentVersion(def.ComponentVersion)); err != nil {
return fmt.Errorf("Error flattening `component_version`: %+v", err)
return fmt.Errorf("failure flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
return fmt.Errorf("failure flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
Expand All @@ -369,7 +381,7 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
edgeNode, err := applicationsClient.Get(ctx, resourceGroup, name, name)
if err != nil {
if !utils.ResponseWasNotFound(edgeNode.Response) {
return fmt.Errorf("Error reading edge node for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("failure reading edge node for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}
}

Expand All @@ -378,13 +390,20 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
}

if err := d.Set("roles", flattenedRoles); err != nil {
return fmt.Errorf("Error flattening `roles`: %+v", err)
return fmt.Errorf("failure flattening `roles`: %+v", err)
}

httpEndpoint := azure.FindHDInsightConnectivityEndpoint("HTTPS", props.ConnectivityEndpoints)
d.Set("https_endpoint", httpEndpoint)
sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
d.Set("ssh_endpoint", sshEndpoint)

monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("failure reading monitor configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("monitor", flattenHDInsightMonitoring(monitor))
}

return tags.FlattenAndSet(d, resp.Tags)
Expand Down Expand Up @@ -473,7 +492,7 @@ func hdInsightWaitForReadyRefreshFunc(ctx context.Context, client *hdinsight.Clu
return func() (interface{}, string, error) {
res, err := client.Get(ctx, resourceGroupName, name)
if err != nil {
return nil, "Error", fmt.Errorf("Error issuing read request in hdInsightWaitForReadyRefreshFunc to Hadoop Cluster %q (Resource Group %q): %s", name, resourceGroupName, err)
return nil, "Error", fmt.Errorf("failure issuing read request in hdInsightWaitForReadyRefreshFunc to Hadoop Cluster %q (Resource Group %q): %s", name, resourceGroupName, err)
}
if props := res.Properties; props != nil {
if state := props.ClusterState; state != nil {
Expand Down
Loading