From a32d2fb9df8c4af94f40bfa7631abf89283878c9 Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Thu, 11 Mar 2021 15:02:59 -0500 Subject: [PATCH 1/3] Add basic prom metrics --- bootstrap/package.json | 1 + bootstrap/src/lib/metrics/index.ts | 21 +++++++++++++++++++++ yarn.lock | 19 +++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 bootstrap/src/lib/metrics/index.ts diff --git a/bootstrap/package.json b/bootstrap/package.json index 9ab27f54c8..23ecd66bcd 100644 --- a/bootstrap/package.json +++ b/bootstrap/package.json @@ -22,6 +22,7 @@ "express": "^4.17.1", "lru-cache": "^6.0.0", "object-hash": "^2.0.3", + "prom-client": "^13.1.0", "promise-timeout": "^1.3.0", "redis": "^3.0.2", "uuid": "^8.3.0" diff --git a/bootstrap/src/lib/metrics/index.ts b/bootstrap/src/lib/metrics/index.ts new file mode 100644 index 0000000000..736f49d1c0 --- /dev/null +++ b/bootstrap/src/lib/metrics/index.ts @@ -0,0 +1,21 @@ +import * as client from 'prom-client' + +const collectDefaultMetrics = client.collectDefaultMetrics +const Registry = client.Registry +const register = new Registry() + +collectDefaultMetrics({ register }) + +export const httpRequestsTotal = new client.Counter({ + name: 'http_requests_total', + help: 'The number of http requests this external adapter has serviced for its entire uptime', + labelNames: ['method', 'statusCode', 'apiKey', 'retry'] as const, +}) + +export const httpRequestDurationSeconds = new client.Histogram({ + name: 'http_request_duration_seconds', + help: 'A histogram bucket of the distribution of http request durations', + // we should tune these as we collect data, this is the default + // bucket distribution that prom comes with + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], +}) diff --git a/yarn.lock b/yarn.lock index 709e9a0c9f..ab44b4e99e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2235,6 +2235,11 @@ bindings@^1.3.0, bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" +bintrees@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524" + integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ= + bip174@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/bip174/-/bip174-2.0.1.tgz#39cf8ca99e50ce538fb762589832f4481d07c254" @@ -6886,6 +6891,13 @@ progress@^2.0.0: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +prom-client@^13.1.0: + version "13.1.0" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-13.1.0.tgz#1185caffd8691e28d32e373972e662964e3dba45" + integrity sha512-jT9VccZCWrJWXdyEtQddCDszYsiuWj5T0ekrPszi/WEegj3IZy6Mm09iOOVM86A4IKMWq8hZkT2dD9MaSe+sng== + dependencies: + tdigest "^0.1.1" + promise-timeout@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/promise-timeout/-/promise-timeout-1.3.0.tgz#d1c78dd50a607d5f0a5207410252a3a0914e1014" @@ -7991,6 +8003,13 @@ tar@^4.0.2: safe-buffer "^5.1.2" yallist "^3.0.3" +tdigest@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.1.tgz#2e3cb2c39ea449e55d1e6cd91117accca4588021" + integrity sha1-Ljyyw56kSeVdHmzZEReszKRYgCE= + dependencies: + bintrees "1.0.1" + teeny-request@^7.0.0: version "7.0.1" resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-7.0.1.tgz#bdd41fdffea5f8fbc0d29392cb47bec4f66b2b4c" From f644f9ed95b7ad73bf6501bf8d11d48ef7b3e2d3 Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Tue, 16 Mar 2021 18:27:34 -0400 Subject: [PATCH 2/3] Hook up prom metrics to http server implementation --- bootstrap/README.md | 6 ++++ bootstrap/src/index.ts | 48 +++++++++++++++++++++++++----- bootstrap/src/lib/metrics/index.ts | 46 ++++++++++++++++++++++++---- bootstrap/src/lib/server.ts | 17 ++++++++++- 4 files changed, 104 insertions(+), 13 deletions(-) diff --git a/bootstrap/README.md b/bootstrap/README.md index 61bf7b8f97..410d4e66ea 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -5,6 +5,12 @@ Bootstrap an external adapter with this package ## Server config - `BASE_URL`: Optional string, Set a base url that is used for setting up routes on the external adapter. Ex. Typically a external adapter is served on the root, so you would make requests to `/`, setting `BASE_URL` to `/coingecko` would instead have requests made to `/coingecko`. Useful when multiple external adapters are being hosted under the same domain, and path mapping is being used to route between them. +## Metrics +A metrics server can be exposed which returns prometheus compatible data on the `/metrics` endpoint on the specified port. Note that this feature is ONLY available when running this application as an http server. Please note that this feature is EXPERIMENTAL. +- `EXPERIMENTAL_METRICS_ENABLED`: Optional bool, defaults to `false`. Set to `true` to enable metrics collection. +- `METRICS_PORT`: Optional number, defaults to `9080`, set to change the port the `/metrics` endpoint is served on +- `METRICS_NAME`: Optional string, defaults to 'N/A', set to apply a label of `NAME` to each metric + ## Caching To cache data, every adapter using the `bootstrap` package, has access to a simple LRU cache that will cache successful 200 responses using SHA1 hash of input as a key. diff --git a/bootstrap/src/index.ts b/bootstrap/src/index.ts index 02d907297a..82e3f7d894 100644 --- a/bootstrap/src/index.ts +++ b/bootstrap/src/index.ts @@ -1,10 +1,11 @@ -import { Requester, logger } from '@chainlink/external-adapter' -import { withCache, defaultOptions, redactOptions } from './lib/cache' -import * as util from './lib/util' -import * as server from './lib/server' -import * as gcp from './lib/gcp' +import { logger, Requester } from '@chainlink/external-adapter' +import { AdapterHealthCheck, AdapterRequest, Execute, ExecuteSync } from '@chainlink/types' import * as aws from './lib/aws' -import { ExecuteSync, AdapterRequest, Execute, AdapterHealthCheck } from '@chainlink/types' +import { defaultOptions, redactOptions, withCache } from './lib/cache' +import * as gcp from './lib/gcp' +import * as metrics from './lib/metrics' +import * as server from './lib/server' +import * as util from './lib/util' export type Middleware = (execute: Execute, options?: O) => Promise @@ -48,7 +49,40 @@ const withLogger: Middleware = async (execute) => async (input: AdapterRequest) } } -const middleware = [withLogger, skipOnError(withCache), withStatusCode] +const withMetrics: Middleware = async (execute) => async (input: AdapterRequest) => { + const recordMetrics = () => { + const labels: Parameters[0] = { + method: 'POST', + } + const end = metrics.httpRequestDurationSeconds.startTimer() + + return (statusCode?: number, type?: metrics.HttpRequestType) => { + labels.type = type + labels.status_code = metrics.normalizeStatusCode(statusCode) + end() + metrics.httpRequestsTotal.labels(labels).inc() + } + } + + const record = recordMetrics() + try { + const result = await execute(input) + record( + result.statusCode, + result.data.maxAge || (result as any).maxAge + ? metrics.HttpRequestType.CACHE_HIT + : metrics.HttpRequestType.DATA_PROVIDER_HIT, + ) + return result + } catch (error) { + record() + throw error + } +} + +const middleware = [withLogger, skipOnError(withCache), withStatusCode].concat( + metrics.METRICS_ENABLED ? [withMetrics] : [], +) // Init all middleware, and return a wrapped execute fn const withMiddleware = async (execute: Execute) => { diff --git a/bootstrap/src/lib/metrics/index.ts b/bootstrap/src/lib/metrics/index.ts index 736f49d1c0..e42bc37bbb 100644 --- a/bootstrap/src/lib/metrics/index.ts +++ b/bootstrap/src/lib/metrics/index.ts @@ -1,15 +1,23 @@ import * as client from 'prom-client' -const collectDefaultMetrics = client.collectDefaultMetrics -const Registry = client.Registry -const register = new Registry() +client.collectDefaultMetrics() +client.register.setDefaultLabels( + // we'll inject both name and versions in + // when EAEE gets merged, because it'll be a lot easier + // to refactor with full type coverage support + { app_name: process.env.METRICS_NAME || 'N/A', app_version: 'N/A' }, +) +export const METRICS_ENABLED = !!process.env.EXPERIMENTAL_METRICS_ENABLED -collectDefaultMetrics({ register }) +export enum HttpRequestType { + CACHE_HIT = 'cacheHit', + DATA_PROVIDER_HIT = 'dataProviderHit', +} export const httpRequestsTotal = new client.Counter({ name: 'http_requests_total', help: 'The number of http requests this external adapter has serviced for its entire uptime', - labelNames: ['method', 'statusCode', 'apiKey', 'retry'] as const, + labelNames: ['method', 'status_code', 'retry', 'type'] as const, }) export const httpRequestDurationSeconds = new client.Histogram({ @@ -19,3 +27,31 @@ export const httpRequestDurationSeconds = new client.Histogram({ // bucket distribution that prom comes with buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], }) + +/** + * Normalizes http status codes. + * + * Returns strings in the format (2|3|4|5)XX. + * + * @author https://github.com/joao-fontenele/express-prometheus-middleware + * @param {!number} status - status code of the requests + * @returns {string} the normalized status code. + */ +export function normalizeStatusCode(status?: number): string { + if (!status) { + return '5XX' + } + + if (status >= 200 && status < 300) { + return '2XX' + } + + if (status >= 300 && status < 400) { + return '3XX' + } + + if (status >= 400 && status < 500) { + return '4XX' + } + return '5XX' +} diff --git a/bootstrap/src/lib/server.ts b/bootstrap/src/lib/server.ts index 5d252103e6..61f082e68d 100644 --- a/bootstrap/src/lib/server.ts +++ b/bootstrap/src/lib/server.ts @@ -1,11 +1,13 @@ import { logger } from '@chainlink/external-adapter' import { AdapterHealthCheck, AdapterResponse, ExecuteSync } from '@chainlink/types' import express from 'express' +import * as client from 'prom-client' import { HTTP_ERROR_NOT_IMPLEMENTED, HTTP_ERROR_UNSUPPORTED_MEDIA_TYPE, HTTP_ERROR_UNSUPPORTED_MEDIA_TYPE_MESSAGE, } from './errors' +import { METRICS_ENABLED } from './metrics' import { toObjectWithNumbers } from './util' const app = express() @@ -23,6 +25,9 @@ export const initHandler = ( execute: ExecuteSync, checkHealth = notImplementedHealthCheck, ) => (): void => { + if (METRICS_ENABLED) { + setupMetricsServer() + } app.use(express.json()) app.post(baseUrl, (req, res) => { @@ -49,8 +54,18 @@ export const initHandler = ( }) app.listen(port, () => logger.info(`Listening on port ${port}!`)) - process.on('SIGINT', () => { process.exit() }) } + +function setupMetricsServer() { + const metricsApp = express() + const metricsPort = process.env.METRICS_PORT || 9080 + + metricsApp.get('/metrics', async (_, res) => { + res.send(await client.register.metrics()) + }) + + metricsApp.listen(metricsPort, () => logger.info(`Monitoring listening on port ${metricsPort}!`)) +} From 32f384c8e7fa2b1acdbcf82d93430eaa3366ca9f Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Wed, 17 Mar 2021 17:54:42 -0400 Subject: [PATCH 3/3] Add bool parsing --- bootstrap/src/lib/metrics/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap/src/lib/metrics/index.ts b/bootstrap/src/lib/metrics/index.ts index e42bc37bbb..743a07a936 100644 --- a/bootstrap/src/lib/metrics/index.ts +++ b/bootstrap/src/lib/metrics/index.ts @@ -1,4 +1,5 @@ import * as client from 'prom-client' +import { parseBool } from '../util' client.collectDefaultMetrics() client.register.setDefaultLabels( @@ -7,7 +8,7 @@ client.register.setDefaultLabels( // to refactor with full type coverage support { app_name: process.env.METRICS_NAME || 'N/A', app_version: 'N/A' }, ) -export const METRICS_ENABLED = !!process.env.EXPERIMENTAL_METRICS_ENABLED +export const METRICS_ENABLED = parseBool(process.env.EXPERIMENTAL_METRICS_ENABLED) export enum HttpRequestType { CACHE_HIT = 'cacheHit',