diff --git a/packages/api/src/client/utils/httpClient.ts b/packages/api/src/client/utils/httpClient.ts index e3743c422145..f82ab7e1f687 100644 --- a/packages/api/src/client/utils/httpClient.ts +++ b/packages/api/src/client/utils/httpClient.ts @@ -82,9 +82,7 @@ export class HttpClient implements IHttpClient { // Attach global signal to this request's controller const onGlobalSignalAbort = controller.abort.bind(controller); const signalGlobal = this.getAbortSignal?.(); - if (signalGlobal) { - signalGlobal.addEventListener("abort", onGlobalSignalAbort); - } + signalGlobal?.addEventListener("abort", onGlobalSignalAbort); const routeId = opts.routeId; // TODO: Should default to "unknown"? const timer = this.metrics?.requestTime.startTimer({routeId}); @@ -113,6 +111,8 @@ export class HttpClient implements IHttpClient { return await getBody(res); } catch (e) { + this.metrics?.requestErrors.inc({routeId}); + if (isAbortedError(e as Error)) { if (signalGlobal?.aborted) { throw new ErrorAborted("REST client"); @@ -121,18 +121,14 @@ export class HttpClient implements IHttpClient { } else { throw Error("Unknown aborted error"); } + } else { + throw e; } - - this.metrics?.errors.inc({routeId}); - - throw e; } finally { timer?.(); clearTimeout(timeout); - if (signalGlobal) { - signalGlobal.removeEventListener("abort", onGlobalSignalAbort); - } + signalGlobal?.removeEventListener("abort", onGlobalSignalAbort); } } } diff --git a/packages/api/src/client/utils/metrics.ts b/packages/api/src/client/utils/metrics.ts index 6d45ce083222..dd663dcdfafa 100644 --- a/packages/api/src/client/utils/metrics.ts +++ b/packages/api/src/client/utils/metrics.ts @@ -1,6 +1,6 @@ export type Metrics = { requestTime: IHistogram<"routeId">; - errors: IGauge<"routeId">; + requestErrors: IGauge<"routeId">; }; type LabelValues = Partial>; diff --git a/packages/lodestar/src/eth1/index.ts b/packages/lodestar/src/eth1/index.ts index f78b7c0ab2d9..2b14641b7b1f 100644 --- a/packages/lodestar/src/eth1/index.ts +++ b/packages/lodestar/src/eth1/index.ts @@ -74,7 +74,8 @@ export class Eth1ForBlockProduction implements IEth1ForBlockProduction { opts: Eth1Options, modules: Eth1DepositDataTrackerModules & Eth1MergeBlockTrackerModules & {eth1Provider?: IEth1Provider} ) { - const eth1Provider = modules.eth1Provider || new Eth1Provider(modules.config, opts, modules.signal); + const eth1Provider = + modules.eth1Provider || new Eth1Provider(modules.config, opts, modules.signal, modules.metrics?.eth1HttpClient); this.eth1DepositDataTracker = opts.disableEth1DepositDataTracker ? null diff --git a/packages/lodestar/src/eth1/interface.ts b/packages/lodestar/src/eth1/interface.ts index ba90dd5ea635..cf11b78bb336 100644 --- a/packages/lodestar/src/eth1/interface.ts +++ b/packages/lodestar/src/eth1/interface.ts @@ -88,7 +88,3 @@ export interface IRpcPayload

{ method: string; params: P; } - -export type ReqOpts = { - timeout?: number; -}; diff --git a/packages/lodestar/src/eth1/provider/eth1Provider.ts b/packages/lodestar/src/eth1/provider/eth1Provider.ts index 607356aa5146..eb7f32f8fa46 100644 --- a/packages/lodestar/src/eth1/provider/eth1Provider.ts +++ b/packages/lodestar/src/eth1/provider/eth1Provider.ts @@ -11,7 +11,7 @@ import {Eth1Block, IEth1Provider} from "../interface.js"; import {Eth1Options} from "../options.js"; import {isValidAddress} from "../../util/address.js"; import {EthJsonRpcBlockRaw} from "../interface.js"; -import {JsonRpcHttpClient} from "./jsonRpcHttpClient.js"; +import {JsonRpcHttpClient, JsonRpcHttpClientMetrics, ReqOpts} from "./jsonRpcHttpClient.js"; import {isJsonRpcTruncatedError, quantityToNum, numToQuantity, dataToBytes} from "./utils.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -37,6 +37,13 @@ interface IEthJsonRpcReturnTypes { }[]; } +// Define static options once to prevent extra allocations +const getBlocksByNumberOpts: ReqOpts = {routeId: "getBlockByNumber_batched"}; +const getBlockByNumberOpts: ReqOpts = {routeId: "getBlockByNumber"}; +const getBlockByHashOpts: ReqOpts = {routeId: "getBlockByHash"}; +const getBlockNumberOpts: ReqOpts = {routeId: "getBlockNumber"}; +const getLogsOpts: ReqOpts = {routeId: "getLogs"}; + export class Eth1Provider implements IEth1Provider { readonly deployBlock: number; private readonly depositContractAddress: string; @@ -45,7 +52,8 @@ export class Eth1Provider implements IEth1Provider { constructor( config: Pick, opts: Pick, - signal?: AbortSignal + signal?: AbortSignal, + metrics?: JsonRpcHttpClientMetrics | null ) { this.deployBlock = opts.depositContractDeployBlock ?? 0; this.depositContractAddress = toHexString(config.DEPOSIT_CONTRACT_ADDRESS); @@ -54,6 +62,7 @@ export class Eth1Provider implements IEth1Provider { // Don't fallback with is truncated error. Throw early and let the retry on this class handle it shouldNotFallback: isJsonRpcTruncatedError, jwtSecret: opts.jwtSecretHex ? fromHex(opts.jwtSecretHex) : undefined, + metrics: metrics, }); } @@ -113,7 +122,8 @@ export class Eth1Provider implements IEth1Provider { return Promise.all( blockRanges.map(([from, to]) => this.rpc.fetchBatch( - linspace(from, to).map((blockNumber) => ({method, params: [numToQuantity(blockNumber), false]})) + linspace(from, to).map((blockNumber) => ({method, params: [numToQuantity(blockNumber), false]})), + getBlocksByNumberOpts ) ) ); @@ -135,25 +145,28 @@ export class Eth1Provider implements IEth1Provider { async getBlockByNumber(blockNumber: number | "latest"): Promise { const method = "eth_getBlockByNumber"; const blockNumberHex = typeof blockNumber === "string" ? blockNumber : numToQuantity(blockNumber); - return await this.rpc.fetch({ - method, + return await this.rpc.fetch( // false = include only transaction roots, not full objects - params: [blockNumberHex, false], - }); + {method, params: [blockNumberHex, false]}, + getBlockByNumberOpts + ); } async getBlockByHash(blockHashHex: string): Promise { const method = "eth_getBlockByHash"; - return await this.rpc.fetch({ - method, + return await this.rpc.fetch( // false = include only transaction roots, not full objects - params: [blockHashHex, false], - }); + {method, params: [blockHashHex, false]}, + getBlockByHashOpts + ); } async getBlockNumber(): Promise { const method = "eth_blockNumber"; - const blockNumberRaw = await this.rpc.fetch({method, params: []}); + const blockNumberRaw = await this.rpc.fetch( + {method, params: []}, + getBlockNumberOpts + ); return parseInt(blockNumberRaw, 16); } @@ -174,7 +187,10 @@ export class Eth1Provider implements IEth1Provider { fromBlock: numToQuantity(options.fromBlock), toBlock: numToQuantity(options.toBlock), }; - const logsRaw = await this.rpc.fetch({method, params: [hexOptions]}); + const logsRaw = await this.rpc.fetch( + {method, params: [hexOptions]}, + getLogsOpts + ); return logsRaw.map((logRaw) => ({ blockNumber: parseInt(logRaw.blockNumber, 16), data: logRaw.data, diff --git a/packages/lodestar/src/eth1/provider/jsonRpcHttpClient.ts b/packages/lodestar/src/eth1/provider/jsonRpcHttpClient.ts index 76a6bdd8c58c..343637d10994 100644 --- a/packages/lodestar/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/lodestar/src/eth1/provider/jsonRpcHttpClient.ts @@ -3,7 +3,8 @@ import fetch from "cross-fetch"; import {ErrorAborted, TimeoutError} from "@chainsafe/lodestar-utils"; -import {IJson, IRpcPayload, ReqOpts} from "../interface.js"; +import {IGauge, IHistogram} from "../../metrics/interface.js"; +import {IJson, IRpcPayload} from "../interface.js"; import {encodeJwtToken} from "./jwt.js"; /** * Limits the amount of response text printed with RPC or parsing errors @@ -24,6 +25,20 @@ interface IRpcResponseError { }; } +export type ReqOpts = { + timeout?: number; + // To label request metrics + routeId?: string; +}; + +export type JsonRpcHttpClientMetrics = { + requestTime: IHistogram<"routeId">; + requestErrors: IGauge<"routeId">; + requestUsedFallbackUrl: IGauge; + activeRequests: IGauge; + configUrlsCount: IGauge; +}; + export interface IJsonRpcHttpClient { fetch(payload: IRpcPayload

, opts?: ReqOpts): Promise; fetchBatch(rpcPayloadArr: IRpcPayload[], opts?: ReqOpts): Promise; @@ -31,13 +46,15 @@ export interface IJsonRpcHttpClient { export class JsonRpcHttpClient implements IJsonRpcHttpClient { private id = 1; + private activeRequests = 0; /** * Optional: If provided, use this jwt secret to HS256 encode and add a jwt token in the * request header which can be authenticated by the RPC server to provide access. * A fresh token is generated on each requests as EL spec mandates the ELs to check * the token freshness +-5 seconds (via `iat` property of the token claim) */ - private jwtSecret?: Uint8Array; + private readonly jwtSecret?: Uint8Array; + private readonly metrics: JsonRpcHttpClientMetrics | null; constructor( private readonly urls: string[], @@ -52,6 +69,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { * and it might deny responses to the RPC requests. */ jwtSecret?: Uint8Array; + metrics?: JsonRpcHttpClientMetrics | null; } ) { // Sanity check for all URLs to be properly defined. Otherwise it will error in loop on fetch @@ -63,7 +81,17 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { throw Error(`JsonRpcHttpClient.urls[${i}] is empty or undefined: ${url}`); } } + this.jwtSecret = opts?.jwtSecret; + this.metrics = opts?.metrics ?? null; + + // Set config metric gauges once + + const metrics = this.metrics; + if (metrics) { + metrics.configUrlsCount.set(urls.length); + metrics.activeRequests.addCollect(() => metrics.activeRequests.set(this.activeRequests)); + } } /** @@ -91,9 +119,13 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { private async fetchJson(json: T, opts?: ReqOpts): Promise { let lastError: Error | null = null; - for (const url of this.urls) { + for (let i = 0; i < this.urls.length; i++) { + if (i > 0) { + this.metrics?.requestUsedFallbackUrl.inc(1); + } + try { - return await this.fetchJsonOneUrl(url, json, opts); + return await this.fetchJsonOneUrl(this.urls[i], json, opts); } catch (e) { if (this.opts?.shouldNotFallback?.(e as Error)) { throw e; @@ -124,15 +156,15 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { // - request to http://missing-url.com/ failed, reason: getaddrinfo ENOTFOUND missing-url.com const controller = new AbortController(); - const timeout = setTimeout(() => { - controller.abort(); - }, opts?.timeout ?? this.opts?.timeout ?? REQUEST_TIMEOUT); + const timeout = setTimeout(() => controller.abort(), opts?.timeout ?? this.opts?.timeout ?? REQUEST_TIMEOUT); const onParentSignalAbort = (): void => controller.abort(); + this.opts?.signal?.addEventListener("abort", onParentSignalAbort, {once: true}); - if (this.opts?.signal) { - this.opts.signal.addEventListener("abort", onParentSignalAbort, {once: true}); - } + // Default to "unknown" to prevent mixing metrics with others. + const routeId = opts?.routeId ?? "unknown"; + const timer = this.metrics?.requestTime.startTimer({routeId}); + this.activeRequests++; try { const headers: Record = {"Content-Type": "application/json"}; @@ -154,9 +186,6 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { body: JSON.stringify(json), headers, signal: controller.signal, - }).finally(() => { - clearTimeout(timeout); - this.opts?.signal?.removeEventListener("abort", onParentSignalAbort); }); const body = await res.text(); @@ -168,6 +197,8 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { return parseJson(body); } catch (e) { + this.metrics?.requestErrors.inc({routeId}); + if (controller.signal.aborted) { // controller will abort on both parent signal abort + timeout of this specific request if (this.opts?.signal?.aborted) { @@ -178,6 +209,12 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { } else { throw e; } + } finally { + timer?.(); + this.activeRequests--; + + clearTimeout(timeout); + this.opts?.signal?.removeEventListener("abort", onParentSignalAbort); } } } diff --git a/packages/lodestar/src/executionEngine/http.ts b/packages/lodestar/src/executionEngine/http.ts index df02b78fff31..4358514a2a75 100644 --- a/packages/lodestar/src/executionEngine/http.ts +++ b/packages/lodestar/src/executionEngine/http.ts @@ -12,7 +12,8 @@ import { QUANTITY, quantityToBigint, } from "../eth1/provider/utils.js"; -import {IJsonRpcHttpClient} from "../eth1/provider/jsonRpcHttpClient.js"; +import {IJsonRpcHttpClient, ReqOpts} from "../eth1/provider/jsonRpcHttpClient.js"; +import {IMetrics} from "../metrics/index.js"; import { ExecutePayloadStatus, ExecutePayloadResponse, @@ -46,6 +47,11 @@ export const defaultExecutionEngineHttpOpts: ExecutionEngineHttpOpts = { timeout: 12000, }; +// Define static options once to prevent extra allocations +const notifyNewPayloadOpts: ReqOpts = {routeId: "notifyNewPayload"}; +const forkchoiceUpdatedV1Opts: ReqOpts = {routeId: "forkchoiceUpdated"}; +const getPayloadOpts: ReqOpts = {routeId: "getPayload"}; + /** * based on Ethereum JSON-RPC API and inherits the following properties of this standard: * - Supported communication protocols (HTTP and WebSocket) @@ -59,14 +65,13 @@ export class ExecutionEngineHttp implements IExecutionEngine { readonly payloadIdCache = new PayloadIdCache(); private readonly rpc: IJsonRpcHttpClient; - constructor(opts: ExecutionEngineHttpOpts, signal: AbortSignal, rpc?: IJsonRpcHttpClient) { - this.rpc = - rpc ?? - new JsonRpcHttpClient(opts.urls, { - signal, - timeout: opts.timeout, - jwtSecret: opts.jwtSecretHex ? fromHex(opts.jwtSecretHex) : undefined, - }); + constructor(opts: ExecutionEngineHttpOpts, signal: AbortSignal, metrics?: IMetrics | null) { + this.rpc = new JsonRpcHttpClient(opts.urls, { + signal, + timeout: opts.timeout, + jwtSecret: opts.jwtSecretHex ? fromHex(opts.jwtSecretHex) : undefined, + metrics: metrics?.executionEnginerHttpClient, + }); } /** @@ -98,10 +103,10 @@ export class ExecutionEngineHttp implements IExecutionEngine { const method = "engine_newPayloadV1"; const serializedExecutionPayload = serializeExecutionPayload(executionPayload); const {status, latestValidHash, validationError} = await this.rpc - .fetch({ - method, - params: [serializedExecutionPayload], - }) + .fetch( + {method, params: [serializedExecutionPayload]}, + notifyNewPayloadOpts + ) // If there are errors by EL like connection refused, internal error, they need to be // treated seperate from being INVALID. For now, just pass the error upstream. .catch((e: Error): EngineApiRpcReturnTypes[typeof method] => { @@ -210,10 +215,10 @@ export class ExecutionEngineHttp implements IExecutionEngine { const { payloadStatus: {status, latestValidHash: _latestValidHash, validationError}, payloadId, - } = await this.rpc.fetch({ - method, - params: [{headBlockHash: headBlockHashData, safeBlockHash, finalizedBlockHash}, apiPayloadAttributes], - }); + } = await this.rpc.fetch( + {method, params: [{headBlockHash: headBlockHashData, safeBlockHash, finalizedBlockHash}, apiPayloadAttributes]}, + forkchoiceUpdatedV1Opts + ); switch (status) { case ExecutePayloadStatus.VALID: @@ -263,10 +268,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { const executionPayloadRpc = await this.rpc.fetch< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] - >({ - method, - params: [payloadId], - }); + >({method, params: [payloadId]}, getPayloadOpts); return parseExecutionPayload(executionPayloadRpc); } diff --git a/packages/lodestar/src/metrics/metrics/lodestar.ts b/packages/lodestar/src/metrics/metrics/lodestar.ts index 447ebe8b9d9e..79b79cfca6fb 100644 --- a/packages/lodestar/src/metrics/metrics/lodestar.ts +++ b/packages/lodestar/src/metrics/metrics/lodestar.ts @@ -960,5 +960,55 @@ export function createLodestarMetrics( help: "Eth1 dynamic follow distance changed by the deposit tracker if blocks are slow", }), }, + + eth1HttpClient: { + requestTime: register.histogram<"routeId">({ + name: "lodestar_eth1_http_client_request_time_seconds", + help: "eth1 JsonHttpClient - histogram or roundtrip request times", + // Provide max resolution on problematic values around 1 second + buckets: [0.1, 0.5, 1, 2, 5, 15], + }), + requestErrors: register.gauge<"routeId">({ + name: "lodestar_eth1_http_client_request_errors_total", + help: "eth1 JsonHttpClient - total count of request errors", + }), + requestUsedFallbackUrl: register.gauge({ + name: "lodestar_eth1_http_client_request_used_fallback_url_total", + help: "eth1 JsonHttpClient - total count of requests on fallback url(s)", + }), + activeRequests: register.gauge({ + name: "lodestar_eth1_http_client_active_requests", + help: "eth1 JsonHttpClient - current count of active requests", + }), + configUrlsCount: register.gauge({ + name: "lodestar_eth1_http_client_config_urls_count", + help: "eth1 JsonHttpClient - static config urls count", + }), + }, + + executionEnginerHttpClient: { + requestTime: register.histogram<"routeId">({ + name: "lodestar_execution_engine_http_client_request_time_seconds", + help: "ExecutionEngineHttp client - histogram or roundtrip request times", + // Provide max resolution on problematic values around 1 second + buckets: [0.1, 0.5, 1, 2, 5, 15], + }), + requestErrors: register.gauge<"routeId">({ + name: "lodestar_execution_engine_http_client_request_errors_total", + help: "ExecutionEngineHttp client - total count of request errors", + }), + requestUsedFallbackUrl: register.gauge({ + name: "lodestar_execution_engine_http_client_request_used_fallback_url_total", + help: "ExecutionEngineHttp client - total count of requests on fallback url(s)", + }), + activeRequests: register.gauge({ + name: "lodestar_execution_engine_http_client_active_requests", + help: "ExecutionEngineHttp client - current count of active requests", + }), + configUrlsCount: register.gauge({ + name: "lodestar_execution_engine_http_client_config_urls_count", + help: "ExecutionEngineHttp client - static config urls count", + }), + }, }; } diff --git a/packages/validator/src/metrics.ts b/packages/validator/src/metrics.ts index 877f33c9f6e2..ffc3f4a3fc84 100644 --- a/packages/validator/src/metrics.ts +++ b/packages/validator/src/metrics.ts @@ -311,9 +311,9 @@ export function getMetrics(register: MetricsRegister, gitData: LodestarGitData) buckets: [0.01, 0.1, 1, 5], }), - errors: register.gauge<{routeId: string}>({ - name: "vc_rest_api_client_errors_total", - help: "Total count of errors calling the REST API client by routeId", + requestErrors: register.gauge<{routeId: string}>({ + name: "vc_rest_api_client_request_errors_total", + help: "Total count of errors on REST API client requests by routeId", labelNames: ["routeId"], }), },