fix: push notifications in background

Sometimes when the alert payload is huge, pushing notification
takes a lot of I/O time and Alertmanager might timeout the request.
This is undesirable since Alertmanager will retry the request but cAlert
will keep sending notifications.

To handle this, it's better to push notifications in it's own goroutine
and send an early response for the request. The downside is that no
error will be sent to Alertmanager if pushing notifications failed, but
that is a fair trade-off. Retries can be implemented at cAlert level (in
a future version).

cmd/handlers.go

@@ -102,14 +102,15 @@ func handleDispatchNotif(w http.ResponseWriter, r *http.Request) {
 	app.lo.WithField("receiver", roomName).Info("dispatching new alert")
 
 	// Dispatch a list of alerts via Notifier.
-	if err := app.notifier.Dispatch(payload.Alerts, roomName); err != nil {
-		app.lo.WithError(err).Error("error dispatching alerts")
-		app.metrics.Increment(`http_request_errors_total{handler="dispatch"}`)
-		sendErrorResponse(w, "Error dispatching alerts.", http.StatusInternalServerError, nil)
-		return
-	}
-
-	app.metrics.Duration(`http_request_duration_seconds{handler="dispatch"}`, now)
+	// If there are a lot of alerts (>=10) to push, G-Chat API can be extremely slow to add messages
+	// to an existing thread. So it's better to enqueue it in background.
+	go func() {
+		if err := app.notifier.Dispatch(payload.Alerts, roomName); err != nil {
+			app.lo.WithError(err).Error("error dispatching alerts")
+			app.metrics.Increment(`http_request_errors_total{handler="dispatch"}`)
+		}
+		app.metrics.Duration(`http_request_duration_seconds{handler="dispatch"}`, now)
+	}()
 
 	sendResponse(w, "dispatched")
 }