add webhooks (#176)

backend/src/analyze-traces.ts

@@ -26,6 +26,7 @@ import {
   insertValueBuilder,
   insertValuesBuilder,
 } from "services/database/utils"
+import { sendWebhookRequests } from "services/webhook"
 
 const getEndpointQuery = (ctx: MetloContext) => `
 SELECT
@@ -179,6 +180,8 @@ const analyze = async (
     )
   }
   await queryRunner.commitTransaction()
+
+  await sendWebhookRequests(ctx, alerts)
 }
 
 const generateEndpoint = async (

backend/src/api/webhook/index.ts

@@ -0,0 +1,43 @@
+import { Response } from "express"
+import ApiResponseHandler from "api-response-handler"
+import { createNewWebhook, deleteWebhook, getWebhooks } from "services/webhook"
+import { MetloRequest } from "types"
+import { CreateWebhookParams } from "@common/types"
+
+export const getWebhooksHandler = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const webhooks = await getWebhooks(req.ctx)
+    await ApiResponseHandler.success(res, webhooks)
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}
+
+export const createWebhookHandler = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const createWebhookParams: CreateWebhookParams = req.body
+    const webhooks = await createNewWebhook(req.ctx, createWebhookParams)
+    await ApiResponseHandler.success(res, webhooks)
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}
+
+export const deleteWebhookHandler = async (
+  req: MetloRequest,
+  res: Response,
+): Promise<void> => {
+  try {
+    const { webhookId } = req.params
+    const webhooks = await deleteWebhook(req.ctx, webhookId)
+    await ApiResponseHandler.success(res, webhooks)
+  } catch (err) {
+    await ApiResponseHandler.error(res, err)
+  }
+}

backend/src/data-source.ts

@@ -15,6 +15,7 @@ import {
   AuthenticationConfig,
   AggregateTraceDataHourly,
   Attack,
+  Webhook,
 } from "models"
 import { runMigration } from "utils"
 import { initMigration1665782029662 } from "migrations/1665782029662-init-migration"
@@ -27,6 +28,7 @@ import { MetloConfig } from "models/metlo-config"
 import { addMetloConfigTable1667599667595 } from "migrations/1667599667595-add-metlo-config-table"
 import { updateDisabledPathsColumnBlockFieldsTable1667606447208 } from "migrations/1667606447208-update-disabledPaths-column-blockFields-table"
 import { removeApiKeyTypeEnum1669778297643 } from "migrations/1669778297643-remove-apiKeyType-enum"
+import { addWebhookTable1670447292139 } from "migrations/1670447292139-add-webhook-table"
 
 export const AppDataSource: DataSource = new DataSource({
   type: "postgres",
@@ -47,6 +49,7 @@ export const AppDataSource: DataSource = new DataSource({
     AggregateTraceDataHourly,
     Attack,
     MetloConfig,
+    Webhook,
   ],
   synchronize: false,
   migrations: [
@@ -59,6 +62,7 @@ export const AppDataSource: DataSource = new DataSource({
     addMetloConfigTable1667599667595,
     updateDisabledPathsColumnBlockFieldsTable1667606447208,
     removeApiKeyTypeEnum1669778297643,
+    addWebhookTable1670447292139,
   ],
   migrationsRun: runMigration,
   logging: false,

backend/src/index.ts

@@ -46,6 +46,11 @@ import {
   getMetloConfigHandler,
   updateMetloConfigHandler,
 } from "api/metlo-config"
+import {
+  createWebhookHandler,
+  deleteWebhookHandler,
+  getWebhooksHandler,
+} from "api/webhook"
 
 const port = process.env.PORT || 8080
 RedisClient.getInstance()
@@ -128,6 +133,10 @@ apiRouter.get("/api/v1/keys/onboarding", getOnboardingKeys)
 apiRouter.put("/api/v1/metlo-config", updateMetloConfigHandler)
 apiRouter.get("/api/v1/metlo-config", getMetloConfigHandler)
 
+apiRouter.get("/api/v1/webhooks", getWebhooksHandler)
+apiRouter.post("/api/v1/webhook", createWebhookHandler)
+apiRouter.delete("/api/v1/webhook/:webhookId", deleteWebhookHandler)
+
 app.use(apiRouter)
 
 const initInstanceSettings = async () => {

backend/src/migrations/1670447292139-add-webhook-table.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class addWebhookTable1670447292139 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `
+      CREATE TABLE IF NOT EXISTS "webhook" (
+        "uuid" uuid NOT NULL DEFAULT uuid_generate_v4(),
+        "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+        "url" character varying NOT NULL,
+        "maxRetries" integer NOT NULL DEFAULT '3',
+        "alertTypes" character varying array NOT NULL DEFAULT '{}',
+        "runs" jsonb NOT NULL DEFAULT '[]',
+        CONSTRAINT "PK_bb57c3c8886ef87304032c70af35b765" PRIMARY KEY ("uuid")
+      )
+    `,
+    )
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE IF EXISTS "webhook"`)
+  }
+}

backend/src/models/index.ts

@@ -12,6 +12,7 @@ import { InstanceSettings } from "./instance-settings"
 import { AuthenticationConfig } from "./authentication-config"
 import { AggregateTraceDataHourly } from "./aggregate-trace-data-hourly"
 import { Attack } from "./attack"
+import { Webhook } from "./webhook"
 
 export type DatabaseModel =
   | ApiEndpoint
@@ -28,6 +29,7 @@ export type DatabaseModel =
   | AuthenticationConfig
   | AggregateTraceDataHourly
   | Attack
+  | Webhook
 
 export {
   ApiEndpoint,
@@ -44,4 +46,5 @@ export {
   AuthenticationConfig,
   AggregateTraceDataHourly,
   Attack,
+  Webhook,
 }

backend/src/models/webhook.ts

@@ -0,0 +1,30 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  PrimaryGeneratedColumn,
+} from "typeorm"
+import { AlertType } from "@common/enums"
+import { WebhookRun } from "@common/types"
+import MetloBaseEntity from "./metlo-base-entity"
+
+@Entity()
+export class Webhook extends MetloBaseEntity {
+  @PrimaryGeneratedColumn("uuid")
+  uuid: string
+
+  @CreateDateColumn({ type: "timestamptz" })
+  createdAt: Date
+
+  @Column({ nullable: false })
+  url: string
+
+  @Column({ type: "integer", nullable: false, default: 3 })
+  maxRetries: number
+
+  @Column({ type: "varchar", array: true, default: [] })
+  alertTypes: AlertType[]
+
+  @Column({ type: "jsonb", nullable: false, default: [] })
+  runs: WebhookRun[]
+}

backend/src/services/webhook/index.ts

@@ -0,0 +1,145 @@
+import axios from "axios"
+import { Brackets } from "typeorm"
+import { Alert, Webhook } from "models"
+import { createQB, getQB, insertValueBuilder } from "services/database/utils"
+import { MetloContext } from "types"
+import { AppDataSource } from "data-source"
+import { CreateWebhookParams } from "@common/types"
+import Error400BadRequest from "errors/error-400-bad-request"
+import Error500InternalServer from "errors/error-500-internal-server"
+
+const urlRegexp = new RegExp(
+  /[(http(s)?):\/\/(www\.)?a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)/,
+)
+
+const validUrl = (url: string) => urlRegexp.test(url)
+
+const delay = (fn: any, ms: number) =>
+  new Promise(resolve => setTimeout(() => resolve(fn()), ms))
+
+const retryRequest = async (fn: any, maxRetries: number) => {
+  const executeRequest = async (attempt: number) => {
+    try {
+      return await fn()
+    } catch (err) {
+      if (err?.response?.status > 400 && attempt <= maxRetries) {
+        return delay(() => executeRequest(attempt + 1), 500)
+      } else {
+        throw err?.response?.data ?? err?.message
+      }
+    }
+  }
+  return executeRequest(1)
+}
+
+export const sendWebhookRequests = async (
+  ctx: MetloContext,
+  alerts: Alert[],
+) => {
+  const queryRunner = AppDataSource.createQueryRunner()
+  try {
+    await queryRunner.connect()
+    for (const alert of alerts) {
+      const webhooks: Webhook[] = await getQB(ctx, queryRunner)
+        .from(Webhook, "webhook")
+        .andWhere(
+          new Brackets(qb => {
+            qb.where(`:type = ANY("alertTypes")`, { type: alert.type }).orWhere(
+              `cardinality("alertTypes") = 0`,
+            )
+          }),
+        )
+        .getRawMany()
+      for (const webhook of webhooks) {
+        let runs = webhook.runs
+        if (runs.length >= 10) {
+          runs = runs.slice(1)
+        }
+        try {
+          await retryRequest(
+            () => axios.post(webhook.url, alert, { timeout: 250 }),
+            webhook.maxRetries,
+          )
+          await getQB(ctx, queryRunner)
+            .update(Webhook)
+            .set({ runs: [...runs, { ok: true, msg: "", payload: alert }] })
+            .andWhere("uuid = :id", { id: webhook.uuid })
+            .execute()
+        } catch (err) {
+          await getQB(ctx, queryRunner)
+            .update(Webhook)
+            .set({ runs: [...runs, { ok: false, msg: err, payload: alert }] })
+            .andWhere("uuid = :id", { id: webhook.uuid })
+            .execute()
+        }
+      }
+    }
+  } catch {
+  } finally {
+    await queryRunner.release()
+  }
+}
+
+export const getWebhooks = async (ctx: MetloContext) => {
+  return await createQB(ctx)
+    .from(Webhook, "webhook")
+    .orderBy(`"createdAt"`, "DESC")
+    .getRawMany()
+}
+
+export const createNewWebhook = async (
+  ctx: MetloContext,
+  createWebhookParams: CreateWebhookParams,
+) => {
+  if (!createWebhookParams.url) {
+    throw new Error400BadRequest("Must provide url for webhook.")
+  }
+  if (!validUrl(createWebhookParams.url)) {
+    throw new Error400BadRequest("Please enter a valid url.")
+  }
+  const queryRunner = AppDataSource.createQueryRunner()
+  try {
+    await queryRunner.connect()
+    const webhook = new Webhook()
+    webhook.url = createWebhookParams.url.trim()
+    if (createWebhookParams.alertTypes?.length > 0) {
+      webhook.alertTypes = createWebhookParams.alertTypes
+    }
+    await insertValueBuilder(ctx, queryRunner, Webhook, webhook).execute()
+    return await getQB(ctx, queryRunner)
+      .from(Webhook, "webhook")
+      .orderBy(`"createdAt"`, "DESC")
+      .getRawMany()
+  } catch {
+    throw new Error500InternalServer(
+      "Encountered error while creating new webhook.",
+    )
+  } finally {
+    await queryRunner.release()
+  }
+}
+
+export const deleteWebhook = async (ctx: MetloContext, webhookId: string) => {
+  if (!webhookId) {
+    throw new Error400BadRequest("Must provide id of webhook to delete.")
+  }
+  const queryRunner = AppDataSource.createQueryRunner()
+  try {
+    await queryRunner.connect()
+    await getQB(ctx, queryRunner)
+      .delete()
+      .from(Webhook, "webhook")
+      .andWhere("uuid = :id", { id: webhookId })
+      .execute()
+    return await getQB(ctx, queryRunner)
+      .from(Webhook, "webhook")
+      .orderBy(`"createdAt"`, "DESC")
+      .getRawMany()
+  } catch {
+    throw new Error500InternalServer(
+      "Encountered error while deleting webhook.",
+    )
+  } finally {
+    await queryRunner.release()
+  }
+}

common/src/types.ts

@@ -377,11 +377,11 @@ export interface STEP_RESPONSE<T extends ConnectionType = ConnectionType> {
     err: string
   }
   data: CONNECTIONS_BASE &
-  (T extends ConnectionType.AWS
-    ? Partial<AWS_CONNECTION & AWS_CONNECTION_MISC & SSH_INFO>
-    : T extends ConnectionType.GCP
-    ? Partial<GCP_CONNECTION & GCP_CONNECTION_MISC>
-    : never)
+    (T extends ConnectionType.AWS
+      ? Partial<AWS_CONNECTION & AWS_CONNECTION_MISC & SSH_INFO>
+      : T extends ConnectionType.GCP
+      ? Partial<GCP_CONNECTION & GCP_CONNECTION_MISC>
+      : never)
   returns?: {
     os_types?: [{ name: string; ami: string }]
     instance_types?: string[]
@@ -532,7 +532,7 @@ export interface DisabledPathSection {
 export interface BlockFieldEntry {
   path: string
   pathRegex: string
-  method: DisableRestMethod,
+  method: DisableRestMethod
   numberParams: number
   disabledPaths: DisabledPathSection
 }
@@ -544,4 +544,24 @@ export interface UpdateMetloConfigParams {
 export interface MetloConfigResp {
   uuid: string
   configString: string
-}
+}
+
+export interface WebhookRun {
+  ok: boolean
+  msg: string
+  payload: Alert
+}
+
+export interface WebhookResp {
+  uuid: string
+  createdAt: Date
+  url: string
+  maxRetries: number
+  alertTypes: AlertType[]
+  runs: WebhookRun[]
+}
+
+export interface CreateWebhookParams {
+  url: string
+  alertTypes: AlertType[]
+}