feat: notification silence

config/crds/mission-control.flanksource.com_incidentrules.yaml

@@ -44,17 +44,15 @@ spec:
                 properties:
                   timeout:
                     description: How long after the health checks have been passing
-                      before, autoclosing the incident.
-                    format: int64
-                    type: integer
+                      before, autoclosing the incident (accepts goduration format)
+                    type: string
                 type: object
               autoResolve:
                 properties:
                   timeout:
                     description: How long after the health checks have been passing
-                      before, autoclosing the incident.
-                    format: int64
-                    type: integer
+                      before, autoclosing the incident (accepts goduration format)
+                    type: string
                 type: object
               breakOnMatch:
                 description: stop processing other incident rules, when matched

db/notifications.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"time"
 
+	extraClausePlugin "github.com/WinterYukky/gorm-extra-clause-plugin"
+	"github.com/WinterYukky/gorm-extra-clause-plugin/exclause"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/query"
@@ -111,3 +113,55 @@ func NotificationSendSummary(ctx context.Context, id string, window time.Duratio
 	err := ctx.DB().Raw(query, id, window).Row().Scan(&earliest, &count)
 	return earliest.Time, count, err
 }
+
+func GetMatchingNotificationSilencesCount(ctx context.Context, resources models.NotificationSilenceResource) (int64, error) {
+	_ = ctx.DB().Use(extraClausePlugin.New())
+
+	query := ctx.DB().Model(&models.NotificationSilence{})
+
+	// Initialize with a false condition,
+	// if no resources are provided, the query won't return all records
+	orClauses := ctx.DB().Where("1 = 0")
+
+	if resources.ConfigID != nil {
+		orClauses = orClauses.Or("config_id = ?", *resources.ConfigID)
+
+		// recursive stuff
+		orClauses = orClauses.Or("(recursive = true AND path_cte.path LIKE '%' || config_id::TEXT || '%')")
+		query = query.Clauses(exclause.NewWith(
+			"path_cte",
+			ctx.DB().Select("path").Model(&models.ConfigItem{}).Where("id = ?", *resources.ConfigID),
+		))
+		query = query.Joins("CROSS JOIN path_cte")
+	}
+
+	if resources.ComponentID != nil {
+		orClauses = orClauses.Or("component_id = ?", *resources.ComponentID)
+
+		// recursive stuff
+		orClauses = orClauses.Or("(recursive = true AND path_cte.path LIKE '%' || component_id::TEXT || '%')")
+		query = query.Clauses(exclause.NewWith(
+			"path_cte",
+			ctx.DB().Select("path").Model(&models.Component{}).Where("id = ?", *resources.ComponentID),
+		))
+		query = query.Joins("CROSS JOIN path_cte")
+	}
+
+	if resources.CanaryID != nil {
+		orClauses = orClauses.Or("canary_id = ?", *resources.CanaryID)
+	}
+
+	if resources.CheckID != nil {
+		orClauses = orClauses.Or("check_id = ?", *resources.CheckID)
+	}
+
+	query = query.Where(orClauses)
+
+	var count int64
+	err := query.Count(&count).Where(`"from" <= NOW()`).Where("until >= NOW()").Where("deleted_at IS NULL").Error
+	if err != nil {
+		return 0, err
+	}
+
+	return count, nil
+}

db/notifications_test.go

@@ -0,0 +1,106 @@
+package db
+
+import (
+	"time"
+
+	"github.com/flanksource/duty/models"
+	"github.com/flanksource/duty/tests/fixtures/dummy"
+	"github.com/google/uuid"
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
+)
+
+var _ = ginkgo.Describe("Notification Silence", ginkgo.Ordered, func() {
+	var silences []models.NotificationSilence
+	ginkgo.BeforeAll(func() {
+		silences = []models.NotificationSilence{
+			{
+				ID:     uuid.New(),
+				From:   time.Now().Add(-time.Hour),
+				Until:  time.Now().Add(time.Hour),
+				Source: models.SourceCRD,
+				NotificationSilenceResource: models.NotificationSilenceResource{
+					ConfigID: lo.ToPtr(dummy.EKSCluster.ID.String()),
+				},
+			},
+			{
+				ID:        uuid.New(),
+				From:      time.Now().Add(-time.Hour),
+				Until:     time.Now().Add(time.Hour),
+				Source:    models.SourceCRD,
+				Recursive: true,
+				NotificationSilenceResource: models.NotificationSilenceResource{
+					ConfigID: lo.ToPtr(dummy.LogisticsAPIDeployment.ID.String()),
+				},
+			},
+			{
+				ID:        uuid.New(),
+				From:      time.Now().Add(-time.Hour),
+				Until:     time.Now().Add(time.Hour),
+				Source:    models.SourceCRD,
+				Recursive: true,
+				NotificationSilenceResource: models.NotificationSilenceResource{
+					ComponentID: lo.ToPtr(dummy.Logistics.ID.String()),
+				},
+			},
+		}
+
+		err := DefaultContext.DB().Create(&silences).Error
+		Expect(err).To(BeNil())
+	})
+
+	ginkgo.Context("non recursive match", func() {
+		ginkgo.It("should match", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.EKSCluster.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(1)))
+		})
+
+		ginkgo.It("should not match", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.KubernetesCluster.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(0)))
+		})
+	})
+
+	ginkgo.Context("config recursive match", func() {
+		ginkgo.It("should match a child", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.LogisticsAPIReplicaSet.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(1)))
+		})
+
+		ginkgo.It("should match a grand child", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.LogisticsAPIPodConfig.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(1)))
+		})
+
+		ginkgo.It("should not match", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ConfigID: lo.ToPtr(dummy.LogisticsUIDeployment.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(0)))
+		})
+	})
+
+	ginkgo.Context("component recursive match", func() {
+		ginkgo.It("should match a child", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ComponentID: lo.ToPtr(dummy.LogisticsAPI.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(1)))
+		})
+
+		ginkgo.It("should match a grand child", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ComponentID: lo.ToPtr(dummy.LogisticsWorker.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(1)))
+		})
+
+		ginkgo.It("should not match", func() {
+			matched, err := GetMatchingNotificationSilencesCount(DefaultContext, models.NotificationSilenceResource{ComponentID: lo.ToPtr(dummy.ClusterComponent.ID.String())})
+			Expect(err).To(BeNil())
+			Expect(matched).To(Equal(int64(0)))
+		})
+	})
+})

db/suite_test.go

@@ -3,6 +3,8 @@ package db
 import (
 	"testing"
 
+	"github.com/flanksource/duty/context"
+	"github.com/flanksource/duty/tests/setup"
 	ginkgo "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -11,3 +13,13 @@ func TestDB(t *testing.T) {
 	RegisterFailHandler(ginkgo.Fail)
 	ginkgo.RunSpecs(t, "DB")
 }
+
+var (
+	DefaultContext context.Context
+)
+
+var _ = ginkgo.BeforeSuite(func() {
+	DefaultContext = setup.BeforeSuiteFn()
+})
+
+var _ = ginkgo.AfterSuite(setup.AfterSuiteFn)

go.mod

@@ -44,6 +44,7 @@ require (
 require (
 	github.com/MicahParks/keyfunc/v2 v2.1.0
 	github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b
+	github.com/WinterYukky/gorm-extra-clause-plugin v0.2.0
 	github.com/aws/aws-sdk-go-v2 v1.30.4
 	github.com/aws/aws-sdk-go-v2/config v1.27.29
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.29
@@ -88,7 +89,6 @@ require (
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/RaveNoX/go-jsonmerge v1.0.0 // indirect
 	github.com/Snawoot/go-http-digest-auth-client v1.1.3 // indirect
-	github.com/WinterYukky/gorm-extra-clause-plugin v0.2.0 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect

notification/controllers.go

@@ -1,8 +1,11 @@
 package notification
 
 import (
+	"encoding/json"
 	"net/http"
 
+	"github.com/flanksource/duty/api"
+	"github.com/flanksource/duty/context"
 	echoSrv "github.com/flanksource/incident-commander/echo"
 	"github.com/flanksource/incident-commander/rbac"
 	"github.com/labstack/echo/v4"
@@ -13,7 +16,24 @@ func init() {
 }
 
 func RegisterRoutes(e *echo.Echo) {
-	e.GET("/notification/events", func(c echo.Context) error {
+	g := e.Group("/notification")
+
+	g.GET("/events", func(c echo.Context) error {
 		return c.JSON(http.StatusOK, EventRing.Get())
 	}, rbac.Authorization(rbac.ObjectMonitor, rbac.ActionRead))
+
+	g.POST("/silence", func(c echo.Context) error {
+		ctx := c.Request().Context().(context.Context)
+
+		var req SilenceSaveRequest
+		if err := json.NewDecoder(c.Request().Body).Decode(&req); err != nil {
+			return err
+		}
+
+		if err := SaveNotificationSilence(ctx, req); err != nil {
+			return api.WriteError(c, err)
+		}
+
+		return nil
+	}, rbac.Authorization(rbac.ObjectNotification, rbac.ActionCreate))
 }

notification/events.go

@@ -128,6 +128,12 @@ func (t *notificationHandler) addNotificationEvent(ctx context.Context, event mo
 
 	t.Ring.Add(event, celEnv)
 
+	silencedResource := getSilencedResourceFromCelEnv(celEnv)
+	matchingSilences, err := db.GetMatchingNotificationSilencesCount(ctx, silencedResource)
+	if err != nil {
+		return err
+	}
+
 	for _, id := range notificationIDs {
 		n, err := GetNotification(ctx, id)
 		if err != nil {
@@ -182,6 +188,21 @@ func (t *notificationHandler) addNotificationEvent(ctx context.Context, event mo
 				continue
 			}
 
+			if matchingSilences > 0 {
+				ctx.Logger.V(6).Infof("silencing notification for event %s due to %d matching silences", event.ID, matchingSilences)
+
+				if err := ctx.DB().Create(&models.NotificationSendHistory{
+					NotificationID: n.ID,
+					ResourceID:     payload.ID,
+					SourceEvent:    event.Name,
+					Status:         "silenced",
+				}).Error; err != nil {
+					return fmt.Errorf("failed to save silenced notification history: %w", err)
+				}
+
+				return nil
+			}
+
 			newEvent := api.Event{
 				Name:       api.EventNotificationSend,
 				Properties: payload.AsMap(),