Rebase to push doc level changes on latest main changes (#391)

* Document level alerting dev (#272)

* Update license headers (#239)

Signed-off-by: Annie Lee <[email protected]>

* Delete unused properties file. (#262)

Signed-off-by: dblock <[email protected]>

* Update cron-utils (#266)

* Update cron-utils

Signed-off-by: Ashish Agrawal <[email protected]>

* Add last run context to Monitor data model

Signed-off-by: Ashish Agrawal <[email protected]>

Co-authored-by: Annie Lee <[email protected]>
Co-authored-by: Daniel Doubrovkine (dB.) <[email protected]>

* add Update Monitor function

Signed-off-by: Ashish Agrawal <[email protected]>

* fix integ test

Signed-off-by: Ashish Agrawal <[email protected]>

* Implemented draft of Finding data model, a new Input type, and some basic unit tests. (#260)

* Implemented draft of Finding data model, and some basic unit tests for it.

Signed-off-by: AWSHurneyt <[email protected]>

* POC for doc-level-alerting (#277)

Signed-off-by: Sriram <[email protected]>

* Add connection to triggers for doc level alerting (#316)

Signed-off-by: Ashish Agrawal <[email protected]>

* CRUD APIs integration Tests and validation"conflict resolved" (#362)

Signed-off-by: charliezhangaws <[email protected]>

* Seggregate monitor runner logic for separation of concerns (#363)

* Refactor monitor runner logic for separation of concerns and better testability.

Signed-off-by: Saurabh Singh <[email protected]>

* Add action and alert flow and findings schema and additional fixes (#381)

Signed-off-by: Ashish Agrawal <[email protected]>

* Finding Search API (#385)

* Findings search API based on Annie's work

Signed-off-by: Annie Lee <[email protected]>

* Fix Search API and add IT tests

Signed-off-by: Ashish Agrawal <[email protected]>

Co-authored-by: Annie Lee <[email protected]>

* Fix integ tests and minor issues from doc level changes

Signed-off-by: Ashish Agrawal <[email protected]>

Co-authored-by: Annie Lee <[email protected]>
Co-authored-by: Daniel Doubrovkine (dB.) <[email protected]>
Co-authored-by: AWSHurneyt <[email protected]>
Co-authored-by: Sriram <[email protected]>
Co-authored-by: charliezhangaws <[email protected]>
Co-authored-by: Saurabh Singh <[email protected]>
Co-authored-by: Annie Lee <[email protected]>

alerting/src/main/kotlin/org/opensearch/alerting/AlertService.kt

@@ -25,10 +25,12 @@ import org.opensearch.alerting.model.ActionRunResult
 import org.opensearch.alerting.model.AggregationResultBucket
 import org.opensearch.alerting.model.Alert
 import org.opensearch.alerting.model.BucketLevelTrigger
+import org.opensearch.alerting.model.DocumentLevelTriggerRunResult
 import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.QueryLevelTriggerRunResult
 import org.opensearch.alerting.model.Trigger
 import org.opensearch.alerting.model.action.AlertCategory
+import org.opensearch.alerting.script.DocumentLevelTriggerExecutionContext
 import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext
 import org.opensearch.alerting.util.IndexUtils
 import org.opensearch.alerting.util.getBucketKeysHash
@@ -166,6 +168,29 @@ class AlertService(
         }
     }
 
+    // TODO: clean this up so it follows the proper alert management for doc monitors
+    fun composeDocLevelAlert(
+        findings: List<String>,
+        relatedDocIds: List<String>,
+        ctx: DocumentLevelTriggerExecutionContext,
+        result: DocumentLevelTriggerRunResult,
+        alertError: AlertError?
+    ): Alert {
+        val currentTime = Instant.now()
+
+        val actionExecutionResults = result.actionResults.map {
+            ActionExecutionResult(it.key, it.value.executionTime, if (it.value.throttled) 1 else 0)
+        }
+
+        val alertState = if (alertError == null) Alert.State.ACTIVE else Alert.State.ERROR
+        return Alert(
+            monitor = ctx.monitor, trigger = ctx.trigger, startTime = currentTime,
+            lastNotificationTime = currentTime, state = alertState, errorMessage = alertError?.message,
+            actionExecutionResults = actionExecutionResults, schemaVersion = IndexUtils.alertIndexSchemaVersion,
+            findingIds = findings, relatedDocIds = relatedDocIds
+        )
+    }
+
     fun updateActionResultsForBucketLevelAlert(
         currentAlert: Alert,
         actionResults: Map<String, ActionRunResult>,

alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt

@@ -17,6 +17,7 @@ import org.opensearch.alerting.action.GetAlertsAction
 import org.opensearch.alerting.action.GetDestinationsAction
 import org.opensearch.alerting.action.GetEmailAccountAction
 import org.opensearch.alerting.action.GetEmailGroupAction
+import org.opensearch.alerting.action.GetFindingsAction
 import org.opensearch.alerting.action.GetMonitorAction
 import org.opensearch.alerting.action.IndexDestinationAction
 import org.opensearch.alerting.action.IndexEmailAccountAction
@@ -32,13 +33,15 @@ import org.opensearch.alerting.core.ScheduledJobIndices
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsTransportAction
 import org.opensearch.alerting.core.model.ClusterMetricsInput
+import org.opensearch.alerting.core.model.DocLevelMonitorInput
 import org.opensearch.alerting.core.model.ScheduledJob
 import org.opensearch.alerting.core.model.SearchInput
 import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler
 import org.opensearch.alerting.core.schedule.JobScheduler
 import org.opensearch.alerting.core.settings.LegacyOpenDistroScheduledJobSettings
 import org.opensearch.alerting.core.settings.ScheduledJobSettings
 import org.opensearch.alerting.model.BucketLevelTrigger
+import org.opensearch.alerting.model.DocumentLevelTrigger
 import org.opensearch.alerting.model.Monitor
 import org.opensearch.alerting.model.QueryLevelTrigger
 import org.opensearch.alerting.resthandler.RestAcknowledgeAlertAction
@@ -51,6 +54,7 @@ import org.opensearch.alerting.resthandler.RestGetAlertsAction
 import org.opensearch.alerting.resthandler.RestGetDestinationsAction
 import org.opensearch.alerting.resthandler.RestGetEmailAccountAction
 import org.opensearch.alerting.resthandler.RestGetEmailGroupAction
+import org.opensearch.alerting.resthandler.RestGetFindingsAction
 import org.opensearch.alerting.resthandler.RestGetMonitorAction
 import org.opensearch.alerting.resthandler.RestIndexDestinationAction
 import org.opensearch.alerting.resthandler.RestIndexEmailAccountAction
@@ -74,6 +78,7 @@ import org.opensearch.alerting.transport.TransportGetAlertsAction
 import org.opensearch.alerting.transport.TransportGetDestinationsAction
 import org.opensearch.alerting.transport.TransportGetEmailAccountAction
 import org.opensearch.alerting.transport.TransportGetEmailGroupAction
+import org.opensearch.alerting.transport.TransportGetFindingsSearchAction
 import org.opensearch.alerting.transport.TransportGetMonitorAction
 import org.opensearch.alerting.transport.TransportIndexDestinationAction
 import org.opensearch.alerting.transport.TransportIndexEmailAccountAction
@@ -139,10 +144,11 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
         @JvmField val EMAIL_GROUP_BASE_URI = "$DESTINATION_BASE_URI/email_groups"
         @JvmField val LEGACY_OPENDISTRO_EMAIL_ACCOUNT_BASE_URI = "$LEGACY_OPENDISTRO_DESTINATION_BASE_URI/email_accounts"
         @JvmField val LEGACY_OPENDISTRO_EMAIL_GROUP_BASE_URI = "$LEGACY_OPENDISTRO_DESTINATION_BASE_URI/email_groups"
+        @JvmField val FINDING_BASE_URI = "/_plugins/_alerting/findings"
         @JvmField val ALERTING_JOB_TYPES = listOf("monitor")
     }
 
-    lateinit var runner: MonitorRunner
+    lateinit var runner: MonitorRunnerService
     lateinit var scheduler: JobScheduler
     lateinit var sweeper: JobSweeper
     lateinit var scheduledJobIndices: ScheduledJobIndices
@@ -178,7 +184,8 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             RestSearchEmailGroupAction(),
             RestGetEmailGroupAction(),
             RestGetDestinationsAction(),
-            RestGetAlertsAction()
+            RestGetAlertsAction(),
+            RestGetFindingsAction()
         )
     }
 
@@ -202,17 +209,21 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
             ActionPlugin.ActionHandler(SearchEmailGroupAction.INSTANCE, TransportSearchEmailGroupAction::class.java),
             ActionPlugin.ActionHandler(DeleteEmailGroupAction.INSTANCE, TransportDeleteEmailGroupAction::class.java),
             ActionPlugin.ActionHandler(GetDestinationsAction.INSTANCE, TransportGetDestinationsAction::class.java),
-            ActionPlugin.ActionHandler(GetAlertsAction.INSTANCE, TransportGetAlertsAction::class.java)
+            ActionPlugin.ActionHandler(GetAlertsAction.INSTANCE, TransportGetAlertsAction::class.java),
+            ActionPlugin.ActionHandler(GetFindingsAction.INSTANCE, TransportGetFindingsSearchAction::class.java)
+
         )
     }
 
     override fun getNamedXContent(): List<NamedXContentRegistry.Entry> {
         return listOf(
             Monitor.XCONTENT_REGISTRY,
             SearchInput.XCONTENT_REGISTRY,
+            DocLevelMonitorInput.XCONTENT_REGISTRY,
             QueryLevelTrigger.XCONTENT_REGISTRY,
             BucketLevelTrigger.XCONTENT_REGISTRY,
-            ClusterMetricsInput.XCONTENT_REGISTRY
+            ClusterMetricsInput.XCONTENT_REGISTRY,
+            DocumentLevelTrigger.XCONTENT_REGISTRY
         )
     }
 
@@ -232,7 +243,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
         // Need to figure out how to use the OpenSearch DI classes rather than handwiring things here.
         val settings = environment.settings()
         alertIndices = AlertIndices(settings, client, threadPool, clusterService)
-        runner = MonitorRunner
+        runner = MonitorRunnerService
             .registerClusterService(clusterService)
             .registerClient(client)
             .registerNamedXContentRegistry(xContentRegistry)