diff --git a/cmd/handlers.go b/cmd/handlers.go index ae90127..f598740 100644 --- a/cmd/handlers.go +++ b/cmd/handlers.go @@ -102,14 +102,15 @@ func handleDispatchNotif(w http.ResponseWriter, r *http.Request) { app.lo.WithField("receiver", roomName).Info("dispatching new alert") // Dispatch a list of alerts via Notifier. - if err := app.notifier.Dispatch(payload.Alerts, roomName); err != nil { - app.lo.WithError(err).Error("error dispatching alerts") - app.metrics.Increment(`http_request_errors_total{handler="dispatch"}`) - sendErrorResponse(w, "Error dispatching alerts.", http.StatusInternalServerError, nil) - return - } - - app.metrics.Duration(`http_request_duration_seconds{handler="dispatch"}`, now) + // If there are a lot of alerts (>=10) to push, G-Chat API can be extremely slow to add messages + // to an existing thread. So it's better to enqueue it in background. + go func() { + if err := app.notifier.Dispatch(payload.Alerts, roomName); err != nil { + app.lo.WithError(err).Error("error dispatching alerts") + app.metrics.Increment(`http_request_errors_total{handler="dispatch"}`) + } + app.metrics.Duration(`http_request_duration_seconds{handler="dispatch"}`, now) + }() sendResponse(w, "dispatched") } diff --git a/docs/examples/mock_payload.json b/docs/examples/mock_payload.json index 65ab629..31cba10 100644 --- a/docs/examples/mock_payload.json +++ b/docs/examples/mock_payload.json @@ -1,7 +1,587 @@ { - "receiver": "calert", + "receiver": "dev_alerts", "status": "firing", "alerts": [ + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, + { + "status": "firing", + "labels": { + "alertname": "DeadManAnotherSwitch", + "monitor": "docker-host-alpha", + "room": "dev_alerts", + "severity": "deadman" + }, + "annotations": { + "description": "This is a DeadMansSwitch meant to ensure that the entire Alerting", + "instance": "", + "severity": "", + "summary": "Consider running `htop` and check the processes consuming max RAM.", + "title": "This is a dummy alert" + }, + "startsAt": "2022-02-16T10:40:45.228Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://7956fb03d26b:9090/graph?g0.expr=vector%282%29+%3D%3D+2\u0026g0.tab=1", + "fingerprint": "1a956348d0570965" + }, { "status": "firing", "labels": {