fix: better test for notification delay health evaluation

go.mod

@@ -359,6 +359,6 @@ require (
 
 // replace github.com/flanksource/commons => /Users/moshe/go/src/github.com/flanksource/commons
 
-replace github.com/flanksource/duty => ../duty
+// replace github.com/flanksource/duty => ../duty
 
 // replace github.com/flanksource/gomplate/v3 => /Users/moshe/go/src/github.com/flanksource/gomplate

go.sum

@@ -877,6 +877,8 @@ github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z
 github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70=
 github.com/flanksource/commons v1.29.10 h1:T/S95Pl8kASEFvQjQ7fJjTUqeVdhxQXg1vfkULTYFJQ=
 github.com/flanksource/commons v1.29.10/go.mod h1:iTbrXOSp3Spv570Nly97D/U9cQjLZoVlmWCXqWzsvRU=
+github.com/flanksource/duty v1.0.657 h1:SoJY93zKRB5QududnGSguC6fCuSU+T+PHuYmK39qRW8=
+github.com/flanksource/duty v1.0.657/go.mod h1:Oj9zIX94CR2U+nmnt97gNLMrsBWILyIhIBeJynIIgqE=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.24.30 h1:6Y25KnAMX4iCuEXe1C8d1kB2PdV+zD1ZulZrv6DV14Q=
 github.com/flanksource/gomplate/v3 v3.24.30/go.mod h1:/lAM7+fkcCCfghCAjzdCqwgWxN5Ow8Sk6nkdFPjejCE=

notification/events.go

@@ -280,17 +280,26 @@ func sendNotifications(ctx context.Context, events models.Events) models.Events
 		if e.Delay != nil {
 			// This was a delayed notification.
 			// We need to re-evaluate the health of the resource.
+			previousHealth := api.EventToHealth(originalEvent.Name)
 
-			health, err := celEnv.GetResourceHealth(ctx)
+			currentHealth, err := celEnv.GetResourceHealth(ctx)
 			if err != nil {
 				e.SetError(err.Error())
 				failedEvents = append(failedEvents, e)
 				notificationContext.WithError(err)
 				continue
 			}
 
-			if health != api.EventToHealth(originalEvent.Name) {
-				ctx.Logger.V(6).Infof("resource health has changed since the original event")
+			notif, err := GetNotification(ctx, payload.NotificationID.String())
+			if err != nil {
+				e.SetError(err.Error())
+				failedEvents = append(failedEvents, e)
+				notificationContext.WithError(err)
+				continue
+			}
+
+			if !isHealthReportable(notif.Events, currentHealth, previousHealth) {
+				ctx.Logger.V(6).Infof("skipping notification[%s] as health change is not reportable", notif.ID)
 				continue
 			}
 		}
@@ -307,6 +316,24 @@ func sendNotifications(ctx context.Context, events models.Events) models.Events
 	return failedEvents
 }
 
+func isHealthReportable(events []string, previousHealth, currentHealth models.Health) bool {
+	isCurrentHealthInNotification := lo.ContainsBy(events, func(event string) bool {
+		return api.EventToHealth(event) == currentHealth
+	})
+	if !isCurrentHealthInNotification {
+		// Either the notification has changed
+		// or the health of the resource has changed to something that the notification isn't configured for
+		return false
+	}
+
+	if previousHealth == currentHealth {
+		return true
+	}
+
+	healthDegraded := models.WorseHealth(previousHealth, currentHealth) == currentHealth
+	return healthDegraded
+}
+
 // getEnvForEvent gets the environment variables for the given event
 // that'll be passed to the cel expression or to the template renderer as a view.
 func getEnvForEvent(ctx context.Context, event models.Event, properties map[string]string) (*celVariables, error) {

notification/events_test.go

@@ -0,0 +1,49 @@
+package notification
+
+import (
+	"testing"
+
+	"github.com/flanksource/duty/models"
+)
+
+func TestIsHealthReportable(t *testing.T) {
+	tests := []struct {
+		name           string
+		events         []string
+		previousHealth models.Health
+		currentHealth  models.Health
+		expected       bool
+	}{
+		{
+			name:           "health worsened",
+			events:         []string{"config.warning", "config.healthy"},
+			previousHealth: models.HealthHealthy,
+			currentHealth:  models.HealthWarning,
+			expected:       true,
+		},
+		{
+			name:           "health changed and got better",
+			events:         []string{"config.warning", "config.healthy"},
+			previousHealth: models.HealthWarning,
+			currentHealth:  models.HealthHealthy,
+			expected:       false,
+		},
+		{
+			name:           "Current health not in notification",
+			events:         []string{"config.healthy"},
+			previousHealth: models.HealthHealthy,
+			currentHealth:  models.HealthUnhealthy,
+			expected:       false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isHealthReportable(tt.events, tt.previousHealth, tt.currentHealth)
+			if result != tt.expected {
+				t.Errorf("isHealthReportable(%v, %v, %v) = %v; want %v",
+					tt.events, tt.previousHealth, tt.currentHealth, result, tt.expected)
+			}
+		})
+	}
+}