Azure monitor support for azurerm_hdinsight_*_cluster

azurerm/helpers/azure/hdinsight.go

@@ -156,6 +156,34 @@ func SchemaHDInsightsExternalMetastore() *schema.Schema {
 	}
 }
 
+func SchemaHDInsightsMonitor() *schema.Schema {
+	return &schema.Schema{
+		Type:     schema.TypeList,
+		Optional: true,
+		MaxItems: 1,
+		Elem: &schema.Resource{
+			Schema: map[string]*schema.Schema{
+				"log_analytics_workspace_id": {
+					Type:     schema.TypeString,
+					Required: true,
+					ForceNew: false,
+				},
+				"primary_key": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     false,
+					Sensitive:    true,
+					ValidateFunc: validation.StringIsNotEmpty,
+					// Azure doesn't return the key
+					DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
+						return (new == d.Get(k).(string)) && (old == "*****")
+					},
+				},
+			},
+		},
+	}
+}
+
 func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
 	vs := input[0].(map[string]interface{})
 
@@ -253,6 +281,18 @@ func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{}
 	}
 }
 
+func ExpandHDInsightsMonitor(input []interface{}) hdinsight.ClusterMonitoringRequest {
+	vs := input[0].(map[string]interface{})
+
+	workspace := vs["log_analytics_workspace_id"].(string)
+	key := vs["primary_key"].(string)
+
+	return hdinsight.ClusterMonitoringRequest{
+		WorkspaceID: utils.String(workspace),
+		PrimaryKey:  utils.String(key),
+	}
+}
+
 func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
 	enabled := false
 	if v, exists := input["restAuthCredential.isEnabled"]; exists && v != nil {

azurerm/internal/services/hdinsight/client/client.go

@@ -9,6 +9,7 @@ type Client struct {
 	ApplicationsClient   *hdinsight.ApplicationsClient
 	ClustersClient       *hdinsight.ClustersClient
 	ConfigurationsClient *hdinsight.ConfigurationsClient
+	ExtensionsClient     *hdinsight.ExtensionsClient
 }
 
 func NewClient(o *common.ClientOptions) *Client {
@@ -21,9 +22,13 @@ func NewClient(o *common.ClientOptions) *Client {
 	ConfigurationsClient := hdinsight.NewConfigurationsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
 	o.ConfigureClient(&ConfigurationsClient.Client, o.ResourceManagerAuthorizer)
 
+	ExtensionsClient := hdinsight.NewExtensionsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
+	o.ConfigureClient(&ExtensionsClient.Client, o.ResourceManagerAuthorizer)
+
 	return &Client{
 		ApplicationsClient:   &ApplicationsClient,
 		ClustersClient:       &ClustersClient,
 		ConfigurationsClient: &ConfigurationsClient,
+		ExtensionsClient:     &ExtensionsClient,
 	}
 }

azurerm/internal/services/hdinsight/common_hdinsight.go

@@ -106,6 +106,19 @@ func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema
 			}
 		}
 
+		if d.HasChange("monitor") {
+			log.Printf("[DEBUG] Chnage Azure Monitor for the HDInsight %q Cluster", clusterKind)
+			extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+			if v, ok := d.GetOk("monitor"); ok {
+				monitorRaw := v.([]interface{})
+				if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+					return err
+				}
+			} else if err := disableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name); err != nil {
+				return nil
+			}
+		}
+
 		return readFunc(d, meta)
 	}
 }
@@ -325,3 +338,42 @@ func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[stri
 		})
 	}
 }
+
+func flattenHDInsightMonitoring(monitor hdinsight.ClusterMonitoringResponse) []interface{} {
+	if *monitor.ClusterMonitoringEnabled {
+		return []interface{}{
+			map[string]string{
+				"log_analytics_workspace_id": *monitor.WorkspaceID,
+				"primary_key":                "*****",
+			}}
+	}
+
+	return nil
+}
+
+func enableHDInsightMonitoring(ctx context.Context, client *hdinsight.ExtensionsClient, resourceGroup, name string, input []interface{}) error {
+	monitor := azure.ExpandHDInsightsMonitor(input)
+	future, err := client.EnableMonitoring(ctx, resourceGroup, name, monitor)
+	if err != nil {
+		return err
+	}
+
+	if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
+		return fmt.Errorf("Error waiting for enabling monitor for  HDInsight Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+	}
+
+	return nil
+}
+
+func disableHDInsightMonitoring(ctx context.Context, client *hdinsight.ExtensionsClient, resourceGroup, name string) error {
+	future, err := client.DisableMonitoring(ctx, resourceGroup, name)
+	if err != nil {
+		return err
+	}
+
+	if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
+		return fmt.Errorf("Error waiting for disabling monitor for  HDInsight Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+	}
+
+	return nil
+}

azurerm/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go

@@ -180,6 +180,8 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
@@ -309,6 +311,15 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 		}
 	}
 
+	// We can only enable monitoring after creation
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightHadoopClusterRead(d, meta)
 }
 
@@ -400,6 +411,15 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go

@@ -118,6 +118,8 @@ func resourceArmHDInsightHBaseCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
@@ -214,6 +216,15 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightHBaseClusterRead(d, meta)
 }
 
@@ -283,6 +294,15 @@ func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go

@@ -118,6 +118,8 @@ func resourceArmHDInsightInteractiveQueryCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
@@ -214,6 +216,15 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightInteractiveQueryClusterRead(d, meta)
 }
 
@@ -283,6 +294,15 @@ func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, met
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go

@@ -119,6 +119,8 @@ func resourceArmHDInsightKafkaCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
@@ -215,6 +217,15 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightKafkaClusterRead(d, meta)
 }
 
@@ -284,6 +295,15 @@ func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_spark_cluster_resource.go

@@ -118,6 +118,8 @@ func resourceArmHDInsightSparkCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
@@ -214,6 +216,15 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightSparkClusterRead(d, meta)
 }
 
@@ -283,6 +294,15 @@ func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_storm_cluster_resource.go

@@ -117,6 +117,8 @@ func resourceArmHDInsightStormCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
@@ -212,6 +214,15 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightStormClusterRead(d, meta)
 }
 
@@ -281,6 +292,15 @@ func resourceArmHDInsightStormClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)