Add basic prom metrics

bootstrap/README.md

@@ -5,6 +5,12 @@ Bootstrap an external adapter with this package
 ## Server config
 - `BASE_URL`: Optional string, Set a base url that is used for setting up routes on the external adapter. Ex. Typically a external adapter is served on the root, so you would make requests to `/`, setting `BASE_URL` to `/coingecko` would instead have requests made to `/coingecko`. Useful when multiple external adapters are being hosted under the same domain, and path mapping is being used to route between them.
 
+## Metrics
+A metrics server can be exposed which returns prometheus compatible data on the `/metrics` endpoint on the specified port. Note that this feature is ONLY available when running this application as an http server. Please note that this feature is EXPERIMENTAL.
+- `EXPERIMENTAL_METRICS_ENABLED`: Optional bool, defaults to `false`. Set to `true` to enable metrics collection.
+- `METRICS_PORT`: Optional number, defaults to `9080`, set to change the port the `/metrics` endpoint is served on
+- `METRICS_NAME`: Optional string, defaults to 'N/A', set to apply a label of `NAME` to each metric
+
 ## Caching
 
 To cache data, every adapter using the `bootstrap` package, has access to a simple LRU cache that will cache successful 200 responses using SHA1 hash of input as a key.

bootstrap/package.json

@@ -22,6 +22,7 @@
     "express": "^4.17.1",
     "lru-cache": "^6.0.0",
     "object-hash": "^2.0.3",
+    "prom-client": "^13.1.0",
     "promise-timeout": "^1.3.0",
     "redis": "^3.0.2",
     "uuid": "^8.3.0"

bootstrap/src/index.ts

@@ -1,10 +1,11 @@
-import { Requester, logger } from '@chainlink/external-adapter'
-import { withCache, defaultOptions, redactOptions } from './lib/cache'
-import * as util from './lib/util'
-import * as server from './lib/server'
-import * as gcp from './lib/gcp'
+import { logger, Requester } from '@chainlink/external-adapter'
+import { AdapterHealthCheck, AdapterRequest, Execute, ExecuteSync } from '@chainlink/types'
 import * as aws from './lib/aws'
-import { ExecuteSync, AdapterRequest, Execute, AdapterHealthCheck } from '@chainlink/types'
+import { defaultOptions, redactOptions, withCache } from './lib/cache'
+import * as gcp from './lib/gcp'
+import * as metrics from './lib/metrics'
+import * as server from './lib/server'
+import * as util from './lib/util'
 
 export type Middleware<O = any> = (execute: Execute, options?: O) => Promise<Execute>
 
@@ -48,7 +49,40 @@ const withLogger: Middleware = async (execute) => async (input: AdapterRequest)
   }
 }
 
-const middleware = [withLogger, skipOnError(withCache), withStatusCode]
+const withMetrics: Middleware = async (execute) => async (input: AdapterRequest) => {
+  const recordMetrics = () => {
+    const labels: Parameters<typeof metrics.httpRequestsTotal.labels>[0] = {
+      method: 'POST',
+    }
+    const end = metrics.httpRequestDurationSeconds.startTimer()
+
+    return (statusCode?: number, type?: metrics.HttpRequestType) => {
+      labels.type = type
+      labels.status_code = metrics.normalizeStatusCode(statusCode)
+      end()
+      metrics.httpRequestsTotal.labels(labels).inc()
+    }
+  }
+
+  const record = recordMetrics()
+  try {
+    const result = await execute(input)
+    record(
+      result.statusCode,
+      result.data.maxAge || (result as any).maxAge
+        ? metrics.HttpRequestType.CACHE_HIT
+        : metrics.HttpRequestType.DATA_PROVIDER_HIT,
+    )
+    return result
+  } catch (error) {
+    record()
+    throw error
+  }
+}
+
+const middleware = [withLogger, skipOnError(withCache), withStatusCode].concat(
+  metrics.METRICS_ENABLED ? [withMetrics] : [],
+)
 
 // Init all middleware, and return a wrapped execute fn
 const withMiddleware = async (execute: Execute) => {

bootstrap/src/lib/metrics/index.ts

@@ -0,0 +1,58 @@
+import * as client from 'prom-client'
+import { parseBool } from '../util'
+
+client.collectDefaultMetrics()
+client.register.setDefaultLabels(
+  // we'll inject both name and versions in
+  // when EAEE gets merged, because it'll be a lot easier
+  // to refactor with full type coverage support
+  { app_name: process.env.METRICS_NAME || 'N/A', app_version: 'N/A' },
+)
+export const METRICS_ENABLED = parseBool(process.env.EXPERIMENTAL_METRICS_ENABLED)
+
+export enum HttpRequestType {
+  CACHE_HIT = 'cacheHit',
+  DATA_PROVIDER_HIT = 'dataProviderHit',
+}
+
+export const httpRequestsTotal = new client.Counter({
+  name: 'http_requests_total',
+  help: 'The number of http requests this external adapter has serviced for its entire uptime',
+  labelNames: ['method', 'status_code', 'retry', 'type'] as const,
+})
+
+export const httpRequestDurationSeconds = new client.Histogram({
+  name: 'http_request_duration_seconds',
+  help: 'A histogram bucket of the distribution of http request durations',
+  // we should tune these as we collect data, this is the default
+  // bucket distribution that prom comes with
+  buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+})
+
+/**
+ * Normalizes http status codes.
+ *
+ * Returns strings in the format (2|3|4|5)XX.
+ *
+ * @author https://github.com/joao-fontenele/express-prometheus-middleware
+ * @param {!number} status - status code of the requests
+ * @returns {string} the normalized status code.
+ */
+export function normalizeStatusCode(status?: number): string {
+  if (!status) {
+    return '5XX'
+  }
+
+  if (status >= 200 && status < 300) {
+    return '2XX'
+  }
+
+  if (status >= 300 && status < 400) {
+    return '3XX'
+  }
+
+  if (status >= 400 && status < 500) {
+    return '4XX'
+  }
+  return '5XX'
+}

bootstrap/src/lib/server.ts

@@ -1,11 +1,13 @@
 import { logger } from '@chainlink/external-adapter'
 import { AdapterHealthCheck, AdapterResponse, ExecuteSync } from '@chainlink/types'
 import express from 'express'
+import * as client from 'prom-client'
 import {
   HTTP_ERROR_NOT_IMPLEMENTED,
   HTTP_ERROR_UNSUPPORTED_MEDIA_TYPE,
   HTTP_ERROR_UNSUPPORTED_MEDIA_TYPE_MESSAGE,
 } from './errors'
+import { METRICS_ENABLED } from './metrics'
 import { toObjectWithNumbers } from './util'
 
 const app = express()
@@ -23,6 +25,9 @@ export const initHandler = (
   execute: ExecuteSync,
   checkHealth = notImplementedHealthCheck,
 ) => (): void => {
+  if (METRICS_ENABLED) {
+    setupMetricsServer()
+  }
   app.use(express.json())
 
   app.post(baseUrl, (req, res) => {
@@ -49,8 +54,18 @@ export const initHandler = (
   })
 
   app.listen(port, () => logger.info(`Listening on port ${port}!`))
-
   process.on('SIGINT', () => {
     process.exit()
   })
 }
+
+function setupMetricsServer() {
+  const metricsApp = express()
+  const metricsPort = process.env.METRICS_PORT || 9080
+
+  metricsApp.get('/metrics', async (_, res) => {
+    res.send(await client.register.metrics())
+  })
+
+  metricsApp.listen(metricsPort, () => logger.info(`Monitoring listening on port ${metricsPort}!`))
+}

yarn.lock

@@ -2235,6 +2235,11 @@ bindings@^1.3.0, bindings@^1.5.0:
   dependencies:
     file-uri-to-path "1.0.0"
 
+[email protected]:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524"
+  integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=
+
 bip174@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/bip174/-/bip174-2.0.1.tgz#39cf8ca99e50ce538fb762589832f4481d07c254"
@@ -6886,6 +6891,13 @@ progress@^2.0.0:
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
+prom-client@^13.1.0:
+  version "13.1.0"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-13.1.0.tgz#1185caffd8691e28d32e373972e662964e3dba45"
+  integrity sha512-jT9VccZCWrJWXdyEtQddCDszYsiuWj5T0ekrPszi/WEegj3IZy6Mm09iOOVM86A4IKMWq8hZkT2dD9MaSe+sng==
+  dependencies:
+    tdigest "^0.1.1"
+
 promise-timeout@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/promise-timeout/-/promise-timeout-1.3.0.tgz#d1c78dd50a607d5f0a5207410252a3a0914e1014"
@@ -7991,6 +8003,13 @@ tar@^4.0.2:
     safe-buffer "^5.1.2"
     yallist "^3.0.3"
 
+tdigest@^0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021"
+  integrity sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=
+  dependencies:
+    bintrees "1.0.1"
+
 teeny-request@^7.0.0:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-7.0.1.tgz#bdd41fdffea5f8fbc0d29392cb47bec4f66b2b4c"