-
Notifications
You must be signed in to change notification settings - Fork 89
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
basic alerting, for internal use primarily
- Loading branch information
1 parent
92e50b3
commit 8b2c0f0
Showing
8 changed files
with
138 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| package evervigil | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| catalog "github.com/PeerDB-io/peer-flow/connectors/utils/catalog" | ||
| "github.com/jackc/pgx/v5" | ||
| "github.com/jackc/pgx/v5/pgxpool" | ||
| "github.com/nikoksr/notify" | ||
| "github.com/nikoksr/notify/service/slack" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| // alerting service, cool name | ||
| type EverVigil struct { | ||
| notifier *notify.Notify | ||
| catalogPool *pgxpool.Pool | ||
| } | ||
|
|
||
| type slackServiceConfig struct { | ||
| AuthToken string `json:"auth_token"` | ||
| ChannelIDs []string `json:"channel_ids"` | ||
| } | ||
|
|
||
| func NewVigil() (*EverVigil, error) { | ||
| catalogPool, catalogErr := catalog.GetCatalogConnectionPoolFromEnv() | ||
| if catalogErr != nil { | ||
| return nil, fmt.Errorf("error getting catalog connection pool: %w", catalogErr) | ||
| } | ||
|
|
||
| notifier := notify.New() | ||
|
|
||
| rows, err := catalogPool.Query(context.Background(), | ||
| "SELECT service_type,service_config FROM peerdb_stats.alerting_config") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to read everVigil config from catalog: %w", err) | ||
| } | ||
|
|
||
| var serviceType, serviceConfig string | ||
| _, err = pgx.ForEachRow(rows, []any{&serviceType, &serviceConfig}, func() error { | ||
| switch serviceType { | ||
| case "slack": | ||
| var slackServiceConfig slackServiceConfig | ||
| err = json.Unmarshal([]byte(serviceConfig), &slackServiceConfig) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to unmarshal Slack service config: %w", err) | ||
| } | ||
|
|
||
| slackService := slack.New(slackServiceConfig.AuthToken) | ||
| slackService.AddReceivers(slackServiceConfig.ChannelIDs...) | ||
| notifier.UseServices(slackService) | ||
| default: | ||
| return fmt.Errorf("unknown service type: %s", serviceType) | ||
| } | ||
| return nil | ||
| }) | ||
|
|
||
| return &EverVigil{ | ||
| notifier: notifier, | ||
| catalogPool: catalogPool, | ||
| }, nil | ||
| } | ||
|
|
||
| func (ev *EverVigil) Close() { | ||
| if ev.catalogPool != nil { | ||
| ev.catalogPool.Close() | ||
| } | ||
| } | ||
|
|
||
| // Only raises an alert if another alert with the same key hasn't been raised | ||
| // in the past 15 minutes | ||
| func (ev *EverVigil) AlertIf(alertKey string, alertMessage string) { | ||
| row := ev.catalogPool.QueryRow(context.Background(), | ||
| `SELECT created_timestamp FROM peerdb_stats.alerts_v1 WHERE alert_key=$1 | ||
| ORDER BY created_timestamp DESC LIMIT 1`, | ||
| alertKey) | ||
| var createdTimestamp time.Time | ||
| err := row.Scan(&createdTimestamp) | ||
| if err != nil && err != pgx.ErrNoRows { | ||
| logrus.Warnf("failed to send alert: %v", err) | ||
| return | ||
| } | ||
|
|
||
| if time.Since(createdTimestamp) >= 15*time.Minute { | ||
| err = ev.notifier.Send(context.Background(), | ||
| fmt.Sprintf(":rotating_light: *Alert Alert* :rotating_light:: %s since %s", alertKey, | ||
| time.Now().Format("2006-01-02 15:04:05.999999")), alertMessage) | ||
| if err != nil { | ||
| logrus.Warnf("failed to send alert: %v", err) | ||
| return | ||
| } | ||
| _, _ = ev.catalogPool.Exec(context.Background(), | ||
| "INSERT INTO peerdb_stats.alerts_v1(alert_key,alert_message) VALUES($1,$2)", | ||
| alertKey, alertMessage) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| CREATE TABLE IF NOT EXISTS peerdb_stats.alerting_config ( | ||
| id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, | ||
| service_type TEXT NOT NULL CHECK (service_type IN ('slack')), | ||
| service_config JSONB NOT NULL | ||
| ); | ||
|
|
||
| CREATE TABLE IF NOT EXISTS peerdb_stats.alerts_v1 ( | ||
| id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, | ||
| alert_key TEXT NOT NULL, | ||
| alert_level TEXT NOT NULL CHECK (alert_level IN ('critical')) DEFAULT 'critical', | ||
| alert_message TEXT NOT NULL, | ||
| created_timestamp TIMESTAMP DEFAULT now() | ||
| ); |