diff --git a/.changelog/6699.txt b/.changelog/6699.txt new file mode 100644 index 00000000000..7290eee1f28 --- /dev/null +++ b/.changelog/6699.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +vertexai: add fields to `vertex_ai_featurestore_entitytype` to support feature value monitoring +``` diff --git a/google/resource_vertex_ai_featurestore_entitytype.go b/google/resource_vertex_ai_featurestore_entitytype.go index 3f018b92cfb..89262e9ca78 100644 --- a/google/resource_vertex_ai_featurestore_entitytype.go +++ b/google/resource_vertex_ai_featurestore_entitytype.go @@ -64,10 +64,66 @@ If this is populated with [FeaturestoreMonitoringConfig.monitoring_interval] spe MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ + "categorical_threshold_config": { + Type: schema.TypeList, + Optional: true, + Description: `Threshold for categorical features of anomaly detection. This is shared by all types of Featurestore Monitoring for categorical features (i.e. Features with type (Feature.ValueType) BOOL or STRING).`, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "value": { + Type: schema.TypeFloat, + Required: true, + Description: `Specify a threshold value that can trigger the alert. For categorical feature, the distribution distance is calculated by L-inifinity norm. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3.`, + }, + }, + }, + }, + "import_features_analysis": { + Type: schema.TypeList, + Optional: true, + Description: `The config for ImportFeatures Analysis Based Feature Monitoring.`, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "anomaly_detection_baseline": { + Type: schema.TypeString, + Optional: true, + Description: `Defines the baseline to do anomaly detection for feature values imported by each [entityTypes.importFeatureValues][] operation. The value must be one of the values below: +* LATEST_STATS: Choose the later one statistics generated by either most recent snapshot analysis or previous import features analysis. If non of them exists, skip anomaly detection and only generate a statistics. +* MOST_RECENT_SNAPSHOT_STATS: Use the statistics generated by the most recent snapshot analysis if exists. +* PREVIOUS_IMPORT_FEATURES_STATS: Use the statistics generated by the previous import features analysis if exists.`, + }, + "state": { + Type: schema.TypeString, + Optional: true, + Description: `Whether to enable / disable / inherite default hebavior for import features analysis. The value must be one of the values below: +* DEFAULT: The default behavior of whether to enable the monitoring. EntityType-level config: disabled. +* ENABLED: Explicitly enables import features analysis. EntityType-level config: by default enables import features analysis for all Features under it. +* DISABLED: Explicitly disables import features analysis. EntityType-level config: by default disables import features analysis for all Features under it.`, + }, + }, + }, + }, + "numerical_threshold_config": { + Type: schema.TypeList, + Optional: true, + Description: `Threshold for numerical features of anomaly detection. This is shared by all objectives of Featurestore Monitoring for numerical features (i.e. Features with type (Feature.ValueType) DOUBLE or INT64).`, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "value": { + Type: schema.TypeFloat, + Required: true, + Description: `Specify a threshold value that can trigger the alert. For numerical feature, the distribution distance is calculated by Jensen–Shannon divergence. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3.`, + }, + }, + }, + }, "snapshot_analysis": { Type: schema.TypeList, Optional: true, - Description: `Configuration of how features in Featurestore are monitored.`, + Description: `The config for Snapshot Analysis Based Feature Monitoring.`, MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -77,6 +133,19 @@ If this is populated with [FeaturestoreMonitoringConfig.monitoring_interval] spe Description: `The monitoring schedule for snapshot analysis. For EntityType-level config: unset / disabled = true indicates disabled by default for Features under it; otherwise by default enable snapshot analysis monitoring with monitoringInterval for Features under it.`, Default: false, }, + "monitoring_interval_days": { + Type: schema.TypeInt, + Optional: true, + Description: `Configuration of the snapshot analysis based monitoring pipeline running interval. The value indicates number of days. The default value is 1. +If both FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days and [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval][] are set when creating/updating EntityTypes/Features, FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days will be used.`, + Default: 1, + }, + "staleness_days": { + Type: schema.TypeInt, + Optional: true, + Description: `Customized export features time window for snapshot analysis. Unit is one day. The default value is 21 days. Minimum value is 1 day. Maximum value is 4000 days.`, + Default: 21, + }, }, }, }, @@ -403,6 +472,12 @@ func flattenVertexAIFeaturestoreEntitytypeMonitoringConfig(v interface{}, d *sch transformed := make(map[string]interface{}) transformed["snapshot_analysis"] = flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(original["snapshotAnalysis"], d, config) + transformed["import_features_analysis"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(original["importFeaturesAnalysis"], d, config) + transformed["numerical_threshold_config"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(original["numericalThresholdConfig"], d, config) + transformed["categorical_threshold_config"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(original["categoricalThresholdConfig"], d, config) return []interface{}{transformed} } func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v interface{}, d *schema.ResourceData, config *Config) interface{} { @@ -416,12 +491,107 @@ func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v int transformed := make(map[string]interface{}) transformed["disabled"] = flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(original["disabled"], d, config) + transformed["monitoring_interval_days"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(original["monitoringIntervalDays"], d, config) + transformed["staleness_days"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(original["stalenessDays"], d, config) return []interface{}{transformed} } func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(v interface{}, d *schema.ResourceData, config *Config) interface{} { return v } +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(v interface{}, d *schema.ResourceData, config *Config) interface{} { + // Handles the string fixed64 format + if strVal, ok := v.(string); ok { + if intVal, err := stringToFixed64(strVal); err == nil { + return intVal + } + } + + // number values are represented as float64 + if floatVal, ok := v.(float64); ok { + intVal := int(floatVal) + return intVal + } + + return v // let terraform core handle it otherwise +} + +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(v interface{}, d *schema.ResourceData, config *Config) interface{} { + // Handles the string fixed64 format + if strVal, ok := v.(string); ok { + if intVal, err := stringToFixed64(strVal); err == nil { + return intVal + } + } + + // number values are represented as float64 + if floatVal, ok := v.(float64); ok { + intVal := int(floatVal) + return intVal + } + + return v // let terraform core handle it otherwise +} + +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(v interface{}, d *schema.ResourceData, config *Config) interface{} { + if v == nil { + return nil + } + original := v.(map[string]interface{}) + if len(original) == 0 { + return nil + } + transformed := make(map[string]interface{}) + transformed["state"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(original["state"], d, config) + transformed["anomaly_detection_baseline"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(original["anomalyDetectionBaseline"], d, config) + return []interface{}{transformed} +} +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} { + if v == nil { + return nil + } + original := v.(map[string]interface{}) + if len(original) == 0 { + return nil + } + transformed := make(map[string]interface{}) + transformed["value"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(original["value"], d, config) + return []interface{}{transformed} +} +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} { + if v == nil { + return nil + } + original := v.(map[string]interface{}) + if len(original) == 0 { + return nil + } + transformed := make(map[string]interface{}) + transformed["value"] = + flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(original["value"], d, config) + return []interface{}{transformed} +} +func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + func expandVertexAIFeaturestoreEntitytypeLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) { if v == nil { return map[string]string{}, nil @@ -449,6 +619,27 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfig(v interface{}, d Terra transformed["snapshotAnalysis"] = transformedSnapshotAnalysis } + transformedImportFeaturesAnalysis, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(original["import_features_analysis"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedImportFeaturesAnalysis); val.IsValid() && !isEmptyValue(val) { + transformed["importFeaturesAnalysis"] = transformedImportFeaturesAnalysis + } + + transformedNumericalThresholdConfig, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(original["numerical_threshold_config"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedNumericalThresholdConfig); val.IsValid() && !isEmptyValue(val) { + transformed["numericalThresholdConfig"] = transformedNumericalThresholdConfig + } + + transformedCategoricalThresholdConfig, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(original["categorical_threshold_config"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedCategoricalThresholdConfig); val.IsValid() && !isEmptyValue(val) { + transformed["categoricalThresholdConfig"] = transformedCategoricalThresholdConfig + } + return transformed, nil } @@ -468,6 +659,20 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v inte transformed["disabled"] = transformedDisabled } + transformedMonitoringIntervalDays, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(original["monitoring_interval_days"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedMonitoringIntervalDays); val.IsValid() && !isEmptyValue(val) { + transformed["monitoringIntervalDays"] = transformedMonitoringIntervalDays + } + + transformedStalenessDays, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(original["staleness_days"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedStalenessDays); val.IsValid() && !isEmptyValue(val) { + transformed["stalenessDays"] = transformedStalenessDays + } + return transformed, nil } @@ -475,6 +680,94 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisable return v, nil } +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedState, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(original["state"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedState); val.IsValid() && !isEmptyValue(val) { + transformed["state"] = transformedState + } + + transformedAnomalyDetectionBaseline, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(original["anomaly_detection_baseline"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedAnomalyDetectionBaseline); val.IsValid() && !isEmptyValue(val) { + transformed["anomalyDetectionBaseline"] = transformedAnomalyDetectionBaseline + } + + return transformed, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedValue, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(original["value"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedValue); val.IsValid() && !isEmptyValue(val) { + transformed["value"] = transformedValue + } + + return transformed, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedValue, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(original["value"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedValue); val.IsValid() && !isEmptyValue(val) { + transformed["value"] = transformedValue + } + + return transformed, nil +} + +func expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func resourceVertexAIFeaturestoreEntitytypeEncoder(d *schema.ResourceData, meta interface{}, obj map[string]interface{}) (map[string]interface{}, error) { if v, ok := d.GetOk("featurestore"); ok { re := regexp.MustCompile("projects/(.+)/locations/(.+)/featurestores/(.+)$") diff --git a/google/resource_vertex_ai_featurestore_entitytype_generated_test.go b/google/resource_vertex_ai_featurestore_entitytype_generated_test.go index 5c8b4ff129e..32a18e5673f 100644 --- a/google/resource_vertex_ai_featurestore_entitytype_generated_test.go +++ b/google/resource_vertex_ai_featurestore_entitytype_generated_test.go @@ -73,6 +73,23 @@ resource "google_vertex_ai_featurestore_entitytype" "entity" { foo = "bar" } featurestore = google_vertex_ai_featurestore.featurestore.id + monitoring_config { + snapshot_analysis { + disabled = false + monitoring_interval_days = 1 + staleness_days = 21 + } + numerical_threshold_config { + value = 0.8 + } + categorical_threshold_config { + value = 10.0 + } + import_features_analysis { + state = "ENABLED" + anomaly_detection_baseline = "PREVIOUS_IMPORT_FEATURES_STATS" + } + } } `, context) } diff --git a/website/docs/r/vertex_ai_featurestore_entitytype.html.markdown b/website/docs/r/vertex_ai_featurestore_entitytype.html.markdown index 4c1e00c8af6..e34b13c5025 100644 --- a/website/docs/r/vertex_ai_featurestore_entitytype.html.markdown +++ b/website/docs/r/vertex_ai_featurestore_entitytype.html.markdown @@ -53,6 +53,23 @@ resource "google_vertex_ai_featurestore_entitytype" "entity" { foo = "bar" } featurestore = google_vertex_ai_featurestore.featurestore.id + monitoring_config { + snapshot_analysis { + disabled = false + monitoring_interval_days = 1 + staleness_days = 21 + } + numerical_threshold_config { + value = 0.8 + } + categorical_threshold_config { + value = 10.0 + } + import_features_analysis { + state = "ENABLED" + anomaly_detection_baseline = "PREVIOUS_IMPORT_FEATURES_STATS" + } + } } ``` ## Example Usage - Vertex Ai Featurestore Entitytype With Beta Fields @@ -86,6 +103,14 @@ resource "google_vertex_ai_featurestore_entitytype" "entity" { disabled = false monitoring_interval = "86400s" } + + categorical_threshold_config { + value = 0.3 + } + + numerical_threshold_config { + value = 0.3 + } } } ``` @@ -122,9 +147,24 @@ The following arguments are supported: * `snapshot_analysis` - (Optional) - Configuration of how features in Featurestore are monitored. + The config for Snapshot Analysis Based Feature Monitoring. Structure is [documented below](#nested_snapshot_analysis). +* `import_features_analysis` - + (Optional) + The config for ImportFeatures Analysis Based Feature Monitoring. + Structure is [documented below](#nested_import_features_analysis). + +* `numerical_threshold_config` - + (Optional) + Threshold for numerical features of anomaly detection. This is shared by all objectives of Featurestore Monitoring for numerical features (i.e. Features with type (Feature.ValueType) DOUBLE or INT64). + Structure is [documented below](#nested_numerical_threshold_config). + +* `categorical_threshold_config` - + (Optional) + Threshold for categorical features of anomaly detection. This is shared by all types of Featurestore Monitoring for categorical features (i.e. Features with type (Feature.ValueType) BOOL or STRING). + Structure is [documented below](#nested_categorical_threshold_config). + The `snapshot_analysis` block supports: @@ -137,6 +177,43 @@ The following arguments are supported: Configuration of the snapshot analysis based monitoring pipeline running interval. The value is rolled up to full day. A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". +* `monitoring_interval_days` - + (Optional) + Configuration of the snapshot analysis based monitoring pipeline running interval. The value indicates number of days. The default value is 1. + If both FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days and [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval][] are set when creating/updating EntityTypes/Features, FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days will be used. + +* `staleness_days` - + (Optional) + Customized export features time window for snapshot analysis. Unit is one day. The default value is 21 days. Minimum value is 1 day. Maximum value is 4000 days. + +The `import_features_analysis` block supports: + +* `state` - + (Optional) + Whether to enable / disable / inherite default hebavior for import features analysis. The value must be one of the values below: + * DEFAULT: The default behavior of whether to enable the monitoring. EntityType-level config: disabled. + * ENABLED: Explicitly enables import features analysis. EntityType-level config: by default enables import features analysis for all Features under it. + * DISABLED: Explicitly disables import features analysis. EntityType-level config: by default disables import features analysis for all Features under it. + +* `anomaly_detection_baseline` - + (Optional) + Defines the baseline to do anomaly detection for feature values imported by each [entityTypes.importFeatureValues][] operation. The value must be one of the values below: + * LATEST_STATS: Choose the later one statistics generated by either most recent snapshot analysis or previous import features analysis. If non of them exists, skip anomaly detection and only generate a statistics. + * MOST_RECENT_SNAPSHOT_STATS: Use the statistics generated by the most recent snapshot analysis if exists. + * PREVIOUS_IMPORT_FEATURES_STATS: Use the statistics generated by the previous import features analysis if exists. + +The `numerical_threshold_config` block supports: + +* `value` - + (Required) + Specify a threshold value that can trigger the alert. For numerical feature, the distribution distance is calculated by Jensen–Shannon divergence. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3. + +The `categorical_threshold_config` block supports: + +* `value` - + (Required) + Specify a threshold value that can trigger the alert. For categorical feature, the distribution distance is calculated by L-inifinity norm. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3. + ## Attributes Reference In addition to the arguments listed above, the following computed attributes are exported: diff --git a/website/docs/r/vertex_ai_featurestore_entitytype_feature.html.markdown b/website/docs/r/vertex_ai_featurestore_entitytype_feature.html.markdown index ae48cfef2e0..9256e70abef 100644 --- a/website/docs/r/vertex_ai_featurestore_entitytype_feature.html.markdown +++ b/website/docs/r/vertex_ai_featurestore_entitytype_feature.html.markdown @@ -100,6 +100,14 @@ resource "google_vertex_ai_featurestore_entitytype" "entity" { disabled = false monitoring_interval = "86400s" } + + categorical_threshold_config { + value = 0.3 + } + + numerical_threshold_config { + value = 0.3 + } } }