diff --git a/docs/usage/local.md b/docs/usage/local.md index ab02d9f00fd..73d01aa054b 100644 --- a/docs/usage/local.md +++ b/docs/usage/local.md @@ -7,7 +7,7 @@ To quickly test and run Lodestar we recommend starting a local testnet. We recom Run a beacon node, with 8 validators with the following command. ```bash -./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --enr.ip 127.0.0.1 --rootDir --reset +./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --startValidators 0:8 --enr.ip 127.0.0.1 --rootDir --reset ``` `--genesisValidators` and `--genesisTime` define the genesis state of the beacon chain. `--rootDir` defines a path where @@ -25,8 +25,7 @@ This would be used to connect from the second node. Start the second node without starting any validators and connect to the first node by supplying the copied `enr` value: ```bash -./lodestar dev --startValidators 0:0 \ - --genesisValidators 8 --genesisTime 1578787200 \ +./lodestar dev --genesisValidators 8 --genesisTime 1578787200 \ --rootDir /path/to/node2 \ --port 9001 \ --api.rest.port 9597 \ @@ -41,7 +40,7 @@ the `--startValidators` option. Passing a value of `0:0` means no validators sho Also, take note that the values of `--genesisValidators` and `--genesisTime` must be the same as the ones passed to the first node in other for the two nodes to have the same beacon chain. -Finally `--port` and `--api.rest.port` are supplied since the default values will already be in use by the first node. +Also `--port` and `--api.rest.port` are supplied since the default values will already be in use by the first node. The `--network.connectToDiscv5Bootnodes` flags needs to be set to true as this is needed to allow connection to boot enrs on local devnet. The exact enr of node to connect to is then supplied via the `--network.discv5.bootEnrs` flag. diff --git a/packages/api/keymanager.d.ts b/packages/api/keymanager.d.ts new file mode 100644 index 00000000000..78162cc2477 --- /dev/null +++ b/packages/api/keymanager.d.ts @@ -0,0 +1 @@ +export * from "./lib/keymanager"; diff --git a/packages/api/keymanager.js b/packages/api/keymanager.js new file mode 100644 index 00000000000..d1e1d777055 --- /dev/null +++ b/packages/api/keymanager.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line @typescript-eslint/no-require-imports +module.exports = require("./lib/keymanager"); diff --git a/packages/api/keymanager_server.d.ts b/packages/api/keymanager_server.d.ts new file mode 100644 index 00000000000..e85c2db4623 --- /dev/null +++ b/packages/api/keymanager_server.d.ts @@ -0,0 +1 @@ +export * from "./lib/keymanager/server"; diff --git a/packages/api/keymanager_server.js b/packages/api/keymanager_server.js new file mode 100644 index 00000000000..24d1348790a --- /dev/null +++ b/packages/api/keymanager_server.js @@ -0,0 +1,2 @@ +// eslint-disable-next-line @typescript-eslint/no-require-imports +module.exports = require("./lib/keymanager/server"); diff --git a/packages/api/src/keymanager/client.ts b/packages/api/src/keymanager/client.ts new file mode 100644 index 00000000000..3092a9850f5 --- /dev/null +++ b/packages/api/src/keymanager/client.ts @@ -0,0 +1,10 @@ +import {IHttpClient, generateGenericJsonClient} from "../client/utils"; +import {Api, ReqTypes, routesData, getReqSerializers, getReturnTypes} from "./routes"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; + +export function getClient(_config: IChainForkConfig, httpClient: IHttpClient): Api { + const reqSerializers = getReqSerializers(); + const returnTypes = getReturnTypes(); + // All routes return JSON, use a client auto-generator + return generateGenericJsonClient(routesData, reqSerializers, returnTypes, httpClient); +} diff --git a/packages/api/src/keymanager/index.ts b/packages/api/src/keymanager/index.ts new file mode 100644 index 00000000000..964ca7d1921 --- /dev/null +++ b/packages/api/src/keymanager/index.ts @@ -0,0 +1,16 @@ +import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {HttpClient, HttpClientOptions} from "../client"; +import {IHttpClient} from "../client/utils"; +import {Api} from "./routes"; +import * as keymanager from "./client"; + +export {ImportStatus, DeletionStatus, KeystoreStr, SlashingProtectionData, PubkeyHex, Api} from "./routes"; + +/** + * REST HTTP client for all keymanager routes + */ +export function getClient(config: IChainForkConfig, opts: HttpClientOptions, httpClient?: IHttpClient): Api { + if (!httpClient) httpClient = new HttpClient(opts); + + return keymanager.getClient(config, httpClient); +} diff --git a/packages/api/src/keymanager/routes.ts b/packages/api/src/keymanager/routes.ts new file mode 100644 index 00000000000..55288fb5ffc --- /dev/null +++ b/packages/api/src/keymanager/routes.ts @@ -0,0 +1,161 @@ +import {ReturnTypes, RoutesData, Schema, reqEmpty, ReqSerializers, ReqEmpty, jsonType} from "../utils"; + +export enum ImportStatus { + /** Keystore successfully decrypted and imported to keymanager permanent storage */ + imported = "imported", + /** Keystore's pubkey is already known to the keymanager */ + duplicate = "duplicate", + /** Any other status different to the above: decrypting error, I/O errors, etc. */ + error = "error", +} + +export enum DeletionStatus { + /** key was active and removed */ + deleted = "deleted", + /** slashing protection data returned but key was not active */ + not_active = "not_active", + /** key was not found to be removed, and no slashing data can be returned */ + not_found = "not_found", + /** unexpected condition meant the key could not be removed (the key was actually found, but we couldn't stop using it) - this would be a sign that making it active elsewhere would almost certainly cause you headaches / slashing conditions etc. */ + error = "error", +} + +/** + * JSON serialized representation of a single keystore in EIP-2335: BLS12-381 Keystore format. + * ``` + * '{"version":4,"uuid":"9f75a3fa-1e5a-49f9-be3d-f5a19779c6fa","path":"m/12381/3600/0/0/0","pubkey":"0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a","crypto":{"kdf":{"function":"pbkdf2","params":{"dklen":32,"c":262144,"prf":"hmac-sha256","salt":"8ff8f22ef522a40f99c6ce07fdcfc1db489d54dfbc6ec35613edf5d836fa1407"},"message":""},"checksum":{"function":"sha256","params":{},"message":"9678a69833d2576e3461dd5fa80f6ac73935ae30d69d07659a709b3cd3eddbe3"},"cipher":{"function":"aes-128-ctr","params":{"iv":"31b69f0ac97261e44141b26aa0da693f"},"message":"e8228bafec4fcbaca3b827e586daad381d53339155b034e5eaae676b715ab05e"}}}' + * ``` + */ +export type KeystoreStr = string; + +/** + * JSON serialized representation of the slash protection data in format defined in EIP-3076: Slashing Protection Interchange Format. + * ``` + * '{"metadata":{"interchange_format_version":"5","genesis_validators_root":"0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2"},"data":[{"pubkey":"0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a","signed_blocks":[],"signed_attestations":[]}]}' + * ``` + */ +export type SlashingProtectionData = string; + +/** + * The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._ + * ``` + * "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a" + * ``` + */ +export type PubkeyHex = string; + +type Statuses = { + status: Status; + message?: string; +}[]; + +type ImportKeystoresReq = { + keystores: KeystoreStr[]; + passwords: string[]; + slashingProtection: SlashingProtectionData; +}; + +type ListKeysResponse = { + validatingPubkey: PubkeyHex; + /** The derivation path (if present in the imported keystore) */ + derivationPath?: string; + /** The key associated with this pubkey cannot be deleted from the API */ + readonly?: boolean; +}; + +export type Api = { + /** + * List all validating pubkeys known to and decrypted by this keymanager binary + * + * https://github.com/ethereum/keymanager-APIs/blob/0c975dae2ac6053c8245ebdb6a9f27c2f114f407/keymanager-oapi.yaml + */ + listKeys(): Promise<{ + data: ListKeysResponse[]; + }>; + + /** + * Import keystores generated by the Eth2.0 deposit CLI tooling. `passwords[i]` must unlock `keystores[i]`. + * + * Users SHOULD send slashing_protection data associated with the imported pubkeys. MUST follow the format defined in + * EIP-3076: Slashing Protection Interchange Format. + * + * @param keystores JSON-encoded keystore files generated with the Launchpad + * @param passwords Passwords to unlock imported keystore files. `passwords[i]` must unlock `keystores[i]` + * @param slashingProtection Slashing protection data for some of the keys of `keystores` + * @returns Status result of each `request.keystores` with same length and order of `request.keystores` + * + * https://github.com/ethereum/keymanager-APIs/blob/0c975dae2ac6053c8245ebdb6a9f27c2f114f407/keymanager-oapi.yaml + */ + importKeystores( + keystoresStr: KeystoreStr[], + passwords: string[], + slashingProtectionStr: SlashingProtectionData + ): Promise<{ + data: Statuses; + }>; + + /** + * DELETE must delete all keys from `request.pubkeys` that are known to the keymanager and exist in its + * persistent storage. Additionally, DELETE must fetch the slashing protection data for the requested keys from + * persistent storage, which must be retained (and not deleted) after the response has been sent. Therefore in the + * case of two identical delete requests being made, both will have access to slashing protection data. + * + * In a single atomic sequential operation the keymanager must: + * 1. Guarantee that key(s) can not produce any more signature; only then + * 2. Delete key(s) and serialize its associated slashing protection data + * + * DELETE should never return a 404 response, even if all pubkeys from request.pubkeys have no extant keystores + * nor slashing protection data. + * + * Slashing protection data must only be returned for keys from `request.pubkeys` for which a + * `deleted` or `not_active` status is returned. + * + * @param pubkeys List of public keys to delete. + * @returns Deletion status of all keys in `request.pubkeys` in the same order. + * + * https://github.com/ethereum/keymanager-APIs/blob/0c975dae2ac6053c8245ebdb6a9f27c2f114f407/keymanager-oapi.yaml + */ + deleteKeystores( + pubkeysHex: string[] + ): Promise<{ + data: Statuses; + slashingProtection: SlashingProtectionData; + }>; +}; + +export const routesData: RoutesData = { + listKeys: {url: "/eth/v1/keystores", method: "GET"}, + importKeystores: {url: "/eth/v1/keystores", method: "POST"}, + deleteKeystores: {url: "/eth/v1/keystores", method: "DELETE"}, +}; + +export type ReqTypes = { + listKeys: ReqEmpty; + importKeystores: {body: ImportKeystoresReq}; + deleteKeystores: {body: {pubkeys: string[]}}; +}; + +export function getReqSerializers(): ReqSerializers { + return { + listKeys: reqEmpty, + importKeystores: { + writeReq: (keystores, passwords, slashingProtection) => ({body: {keystores, passwords, slashingProtection}}), + parseReq: ({body: {keystores, passwords, slashingProtection}}) => [keystores, passwords, slashingProtection], + schema: {body: Schema.Object}, + }, + deleteKeystores: { + writeReq: (pubkeys) => ({body: {pubkeys}}), + parseReq: ({body: {pubkeys}}) => [pubkeys], + schema: {body: Schema.Object}, + }, + }; +} + +/* eslint-disable @typescript-eslint/naming-convention */ +export function getReturnTypes(): ReturnTypes { + return { + listKeys: jsonType(), + importKeystores: jsonType(), + deleteKeystores: jsonType(), + }; +} diff --git a/packages/api/src/keymanager/server.ts b/packages/api/src/keymanager/server.ts new file mode 100644 index 00000000000..02f3fc3b54e --- /dev/null +++ b/packages/api/src/keymanager/server.ts @@ -0,0 +1,8 @@ +import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {ServerRoutes, getGenericJsonServer} from "../server/utils"; +import {Api, ReqTypes, routesData, getReturnTypes, getReqSerializers} from "./routes"; + +export function getRoutes(config: IChainForkConfig, api: Api): ServerRoutes { + // All routes return JSON, use a server auto-generator + return getGenericJsonServer({routesData, getReturnTypes, getReqSerializers}, config, api); +} diff --git a/packages/api/src/utils/types.ts b/packages/api/src/utils/types.ts index 0a54fca39c9..4c26ddcf8d1 100644 --- a/packages/api/src/utils/types.ts +++ b/packages/api/src/utils/types.ts @@ -24,7 +24,7 @@ export type RouteGroupDefinition< export type RouteDef = { url: string; - method: "GET" | "POST"; + method: "GET" | "POST" | "DELETE"; }; export type ReqGeneric = { diff --git a/packages/api/test/unit/keymanager.test.ts b/packages/api/test/unit/keymanager.test.ts new file mode 100644 index 00000000000..3d1f5fc4d12 --- /dev/null +++ b/packages/api/test/unit/keymanager.test.ts @@ -0,0 +1,32 @@ +import {config} from "@chainsafe/lodestar-config/default"; +import {Api, DeletionStatus, ImportStatus, ReqTypes} from "../../src/keymanager/routes"; +import {getClient} from "../../src/keymanager/client"; +import {getRoutes} from "../../src/keymanager/server"; +import {runGenericServerTest} from "../utils/genericServerTest"; + +describe("keymanager", () => { + runGenericServerTest(config, getClient, getRoutes, { + listKeys: { + args: [], + res: { + data: [ + { + validatingPubkey: + // randomly pregenerated pubkey + "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576", + derivationPath: "m/12381/3600/0/0/0", + readonly: false, + }, + ], + }, + }, + importKeystores: { + args: [["key1"], ["pass1"], "slash_protection"], + res: {data: [{status: ImportStatus.imported}]}, + }, + deleteKeystores: { + args: [["key1"]], + res: {data: [{status: DeletionStatus.deleted}], slashingProtection: "slash_protection"}, + }, + }); +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index cb288dc88d2..db0a70279ed 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -51,7 +51,7 @@ "@chainsafe/abort-controller": "^3.0.1", "@chainsafe/bls": "6.0.3", "@chainsafe/bls-keygen": "^0.3.0", - "@chainsafe/bls-keystore": "2.0.0", + "@chainsafe/bls-keystore": "^2.0.0", "@chainsafe/blst": "^0.2.4", "@chainsafe/discv5": "^0.6.7", "@chainsafe/lodestar": "^0.36.0", @@ -59,6 +59,7 @@ "@chainsafe/lodestar-beacon-state-transition": "^0.36.0", "@chainsafe/lodestar-config": "^0.36.0", "@chainsafe/lodestar-db": "^0.36.0", + "@chainsafe/lodestar-keymanager-server": "^0.36.0", "@chainsafe/lodestar-params": "^0.36.0", "@chainsafe/lodestar-types": "^0.36.0", "@chainsafe/lodestar-utils": "^0.36.0", diff --git a/packages/cli/src/cmds/account/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/account/cmds/validator/voluntaryExit.ts index 757f5c4b832..fd95c215c1f 100644 --- a/packages/cli/src/cmds/account/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/account/cmds/validator/voluntaryExit.ts @@ -1,5 +1,4 @@ -import {SlashingProtection, Validator} from "@chainsafe/lodestar-validator"; -import {SignerType} from "@chainsafe/lodestar-validator"; +import {SignerType, SlashingProtection, Validator} from "@chainsafe/lodestar-validator"; import {readdirSync} from "node:fs"; import {LevelDbController} from "@chainsafe/lodestar-db"; import inquirer from "inquirer"; diff --git a/packages/cli/src/cmds/dev/options.ts b/packages/cli/src/cmds/dev/options.ts index cf664e905d9..20dfcea7ccc 100644 --- a/packages/cli/src/cmds/dev/options.ts +++ b/packages/cli/src/cmds/dev/options.ts @@ -2,17 +2,25 @@ import {Options} from "yargs"; import {ICliCommandOptions} from "../../util"; import {beaconOptions, IBeaconArgs} from "../beacon/options"; import {beaconNodeOptions} from "../../options"; +import {IValidatorCliArgs, validatorOptions} from "../validator/options"; +import {KeymanagerArgs, keymanagerOptions} from "../../options/keymanagerOptions"; -interface IDevOwnArgs { +type IDevOwnArgs = { genesisEth1Hash?: string; genesisValidators?: number; startValidators?: string; genesisTime?: number; reset?: boolean; server: string; -} +} & KeymanagerArgs & + Pick; const devOwnOptions: ICliCommandOptions = { + ...keymanagerOptions, + ...{ + importKeystoresPath: validatorOptions["importKeystoresPath"], + importKeystoresPassword: validatorOptions["importKeystoresPassword"], + }, genesisEth1Hash: { description: "If present it will create genesis with this eth1 hash.", type: "string", diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 93c2020f536..6ccb6766bfd 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -1,7 +1,8 @@ import {AbortController} from "@chainsafe/abort-controller"; import {getClient} from "@chainsafe/lodestar-api"; -import {Validator, SlashingProtection, Signer, SignerType} from "@chainsafe/lodestar-validator"; import {LevelDbController} from "@chainsafe/lodestar-db"; +import {SignerType, Signer, SlashingProtection, Validator} from "@chainsafe/lodestar-validator"; +import {KeymanagerServer, KeymanagerApi} from "@chainsafe/lodestar-keymanager-server"; import {getBeaconConfigFromArgs} from "../../config"; import {IGlobalArgs} from "../../options"; import {YargsError, getDefaultGraffiti, initBLS, mkdir, getCliLogger} from "../../util"; @@ -102,4 +103,24 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P onGracefulShutdownCbs.push(async () => await validator.stop()); await validator.start(); + + // Start keymanager API backend + // Only if keymanagerEnabled flag is set to true and at least one keystore path is supplied + const firstImportKeystorePath = args.importKeystoresPath?.[0]; + if (args.keymanagerEnabled && firstImportKeystorePath) { + const keymanagerApi = new KeymanagerApi(logger, validator, firstImportKeystorePath); + + const keymanagerServer = new KeymanagerServer( + { + host: args.keymanagerHost, + port: args.keymanagerPort, + cors: args.keymanagerCors, + isAuthEnabled: args.keymanagerAuthEnabled, + tokenDir: dbPath, + }, + {config, logger, api: keymanagerApi} + ); + onGracefulShutdownCbs.push(() => keymanagerServer.close()); + await keymanagerServer.listen(); + } } diff --git a/packages/cli/src/cmds/validator/keys.ts b/packages/cli/src/cmds/validator/keys.ts index 9da1bf09b29..b4c8e13b1eb 100644 --- a/packages/cli/src/cmds/validator/keys.ts +++ b/packages/cli/src/cmds/validator/keys.ts @@ -5,15 +5,14 @@ import {CoordType, PublicKey, SecretKey} from "@chainsafe/bls"; import {deriveEth2ValidatorKeys, deriveKeyFromMnemonic} from "@chainsafe/bls-keygen"; import {interopSecretKey} from "@chainsafe/lodestar-beacon-state-transition"; import {externalSignerGetKeys} from "@chainsafe/lodestar-validator"; +import {LOCK_FILE_EXT, getLockFile} from "@chainsafe/lodestar-keymanager-server"; import {defaultNetwork, IGlobalArgs} from "../../options"; import {parseRange, stripOffNewlines, YargsError} from "../../util"; -import {getLockFile} from "../../util/lockfile"; import {ValidatorDirManager} from "../../validatorDir"; import {getAccountPaths} from "../account/paths"; import {IValidatorCliArgs} from "./options"; import {fromHexString} from "@chainsafe/ssz"; -const LOCK_FILE_EXT = ".lock"; const depositDataPattern = new RegExp(/^deposit_data-\d+\.json$/gi); export async function getLocalSecretKeys( diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 9be1e4179bd..560f01b3222 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -3,6 +3,7 @@ import {defaultValidatorPaths} from "./paths"; import {accountValidatorOptions, IAccountValidatorArgs} from "../account/cmds/validator/options"; import {logOptions, beaconPathsOptions} from "../beacon/options"; import {IBeaconPaths} from "../beacon/paths"; +import {KeymanagerArgs, keymanagerOptions} from "../../options/keymanagerOptions"; export type IValidatorCliArgs = IAccountValidatorArgs & ILogArgs & { @@ -19,11 +20,12 @@ export type IValidatorCliArgs = IAccountValidatorArgs & interopIndexes?: string; fromMnemonic?: string; mnemonicIndexes?: string; - }; + } & KeymanagerArgs; export const validatorOptions: ICliCommandOptions = { ...accountValidatorOptions, ...logOptions, + ...keymanagerOptions, logFile: beaconPathsOptions.logFile, validatorsDbDir: { @@ -61,6 +63,8 @@ export const validatorOptions: ICliCommandOptions = { type: "string", }, + // HIDDEN INTEROP OPTIONS + // Remote signer externalSignerUrl: { diff --git a/packages/cli/src/config/beaconNodeOptions.ts b/packages/cli/src/config/beaconNodeOptions.ts index 5c9b888b894..559d8e3688d 100644 --- a/packages/cli/src/config/beaconNodeOptions.ts +++ b/packages/cli/src/config/beaconNodeOptions.ts @@ -50,10 +50,6 @@ export class BeaconNodeOptions { set(beaconNodeOptionsPartial: RecursivePartial): void { this.beaconNodeOptions = mergeBeaconNodeOptions(this.beaconNodeOptions, beaconNodeOptionsPartial); } - - writeTo(filepath: string): void { - writeFile(filepath, this.beaconNodeOptions as Json); - } } export function writeBeaconNodeOptions(filename: string, config: Partial): void { diff --git a/packages/cli/src/options/keymanagerOptions.ts b/packages/cli/src/options/keymanagerOptions.ts new file mode 100644 index 00000000000..6257d432a36 --- /dev/null +++ b/packages/cli/src/options/keymanagerOptions.ts @@ -0,0 +1,43 @@ +import {ICliCommandOptions} from "../util"; +import {restApiOptionsDefault} from "@chainsafe/lodestar-keymanager-server"; + +export type KeymanagerArgs = { + keymanagerEnabled?: boolean; + keymanagerAuthEnabled?: boolean; + keymanagerPort?: number; + keymanagerHost?: string; + keymanagerCors?: string; +}; + +export const keymanagerOptions: ICliCommandOptions = { + keymanagerEnabled: { + type: "boolean", + description: "Enable keymanager API server", + default: false, + group: "keymanager", + }, + keymanagerAuthEnabled: { + type: "boolean", + description: "Enable token bearer authentication for keymanager API server", + default: true, + group: "keymanager", + }, + keymanagerPort: { + type: "number", + description: "Set port for keymanager API", + defaultDescription: String(restApiOptionsDefault.port), + group: "keymanager", + }, + keymanagerHost: { + type: "string", + description: "Set host for keymanager API", + defaultDescription: restApiOptionsDefault.host, + group: "keymanager", + }, + keymanagerCors: { + type: "string", + description: "Configures the Access-Control-Allow-Origin CORS header for keymanager API", + defaultDescription: restApiOptionsDefault.cors, + group: "keymanager", + }, +}; diff --git a/packages/cli/src/validatorDir/ValidatorDir.ts b/packages/cli/src/validatorDir/ValidatorDir.ts index 5093d69c1de..f001cae2cec 100644 --- a/packages/cli/src/validatorDir/ValidatorDir.ts +++ b/packages/cli/src/validatorDir/ValidatorDir.ts @@ -3,10 +3,10 @@ import path from "node:path"; import bls, {SecretKey} from "@chainsafe/bls"; import {Keystore} from "@chainsafe/bls-keystore"; import {phase0} from "@chainsafe/lodestar-types"; +import {getLockFile} from "@chainsafe/lodestar-keymanager-server"; import {YargsError, readValidatorPassphrase} from "../util"; import {decodeEth1TxData} from "../depositContract/depositData"; import {add0xPrefix} from "../util/format"; -import {getLockFile} from "../util/lockfile"; import { VOTING_KEYSTORE_FILE, WITHDRAWAL_KEYSTORE_FILE, diff --git a/packages/cli/test/e2e/cmds/beacon.test.ts b/packages/cli/test/e2e/cmds/beacon.test.ts deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/cli/test/e2e/cmds/validator.test.ts b/packages/cli/test/e2e/cmds/validator.test.ts deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/keymanager-server/.mocharc.yaml b/packages/keymanager-server/.mocharc.yaml new file mode 100644 index 00000000000..731bc281fe4 --- /dev/null +++ b/packages/keymanager-server/.mocharc.yaml @@ -0,0 +1,4 @@ +colors: true +require: + - ts-node/register + - ./test/setup.ts diff --git a/packages/keymanager-server/LICENSE b/packages/keymanager-server/LICENSE new file mode 100644 index 00000000000..f49a4e16e68 --- /dev/null +++ b/packages/keymanager-server/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/keymanager-server/README.md b/packages/keymanager-server/README.md new file mode 100644 index 00000000000..264ab1d475c --- /dev/null +++ b/packages/keymanager-server/README.md @@ -0,0 +1,20 @@ +# Lodestar Keymanager server + +[![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr) +![ETH2.0_Spec_Version 1.0.0](https://img.shields.io/badge/ETH2.0_Spec_Version-1.0.0-2e86c1.svg) +![ES Version](https://img.shields.io/badge/ES-2020-yellow) +![Node Version](https://img.shields.io/badge/node-12.x-green) + +> This package is part of [ChainSafe's Lodestar](https://lodestar.chainsafe.io) project + +Typescript implementation of the Eth2.0 keymanager API backend. Follows the standard defined in https://github.com/ethereum/keymanager-APIs + +## Getting started + +- Follow the [installation guide](https://chainsafe.github.io/lodestar/installation) to install Lodestar. +- Quickly try out the whole stack by [starting a local testnet](https://chainsafe.github.io/lodestar/usage). +- View the [typedoc code docs](https://chainsafe.github.io/lodestar/packages). + +## License + +Apache-2.0 [ChainSafe Systems](https://chainsafe.io) diff --git a/packages/keymanager-server/package.json b/packages/keymanager-server/package.json new file mode 100644 index 00000000000..741a30e50f1 --- /dev/null +++ b/packages/keymanager-server/package.json @@ -0,0 +1,65 @@ +{ + "name": "@chainsafe/lodestar-keymanager-server", + "version": "0.36.0", + "description": "A Typescript implementation of the keymanager server", + "author": "ChainSafe Systems", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/ChainSafe/lodestar/issues" + }, + "homepage": "https://github.com/ChainSafe/lodestar#readme", + "main": "lib/index.js", + "files": [ + "lib/**/*.js", + "lib/**/*.js.map", + "lib/**/*.d.ts", + "*.d.ts", + "*.js" + ], + "scripts": { + "clean": "rm -rf lib && rm -f *.tsbuildinfo", + "build": "tsc -p tsconfig.build.json", + "build:release": "yarn clean && yarn run build && yarn run build:typedocs", + "build:lib:watch": "yarn run build:lib --watch", + "build:typedocs": "typedoc --exclude src/index.ts --out typedocs src", + "build:types:watch": "yarn run build:types --watch", + "check-types": "tsc", + "lint": "eslint --color --ext .ts src/ test/", + "lint:fix": "yarn run lint --fix", + "pretest": "yarn run check-types", + "test": "yarn test:unit", + "coverage": "codecov -F lodestar-validator", + "check-readme": "typescript-docs-verifier" + }, + "repository": { + "type": "git", + "url": "git+https://github.com:ChainSafe/lodestar.git" + }, + "keywords": [ + "ethereum", + "eth2", + "beacon", + "blockchain" + ], + "dependencies": { + "@chainsafe/abort-controller": "^3.0.1", + "@chainsafe/bls": "6.0.3", + "@chainsafe/bls-keystore": "^2.0.0", + "@chainsafe/lodestar-api": "^0.36.0", + "@chainsafe/lodestar-config": "^0.36.0", + "@chainsafe/lodestar-db": "^0.36.0", + "@chainsafe/lodestar-params": "^0.36.0", + "@chainsafe/lodestar-types": "^0.36.0", + "@chainsafe/lodestar-utils": "^0.36.0", + "@chainsafe/lodestar-validator": "^0.36.0", + "@chainsafe/ssz": "^0.8.20", + "lockfile": "^1.0.4", + "fastify": "3.15.1", + "fastify-cors": "^6.0.1", + "fastify-bearer-auth": "6.1.0" + }, + "devDependencies": { + "bigint-buffer": "^1.1.5", + "@types/lockfile": "^1.0.1" + } +} diff --git a/packages/keymanager-server/src/impl.ts b/packages/keymanager-server/src/impl.ts new file mode 100644 index 00000000000..ca16d4e2e4d --- /dev/null +++ b/packages/keymanager-server/src/impl.ts @@ -0,0 +1,213 @@ +import fs from "node:fs"; +import path from "node:path"; +import {SecretKey} from "@chainsafe/bls"; +import {Keystore} from "@chainsafe/bls-keystore"; +import { + Api, + DeletionStatus, + ImportStatus, + KeystoreStr, + SlashingProtectionData, +} from "@chainsafe/lodestar-api/keymanager"; +import {fromHexString} from "@chainsafe/ssz"; +import {Interchange, SignerType, Validator} from "@chainsafe/lodestar-validator"; +import {PubkeyHex} from "@chainsafe/lodestar-validator/src/types"; +import {ILogger} from "@chainsafe/lodestar-utils"; +import {LOCK_FILE_EXT, getLockFile} from "./util/lockfile"; + +export const KEY_IMPORTED_PREFIX = "key_imported"; + +export class KeymanagerApi implements Api { + constructor( + private readonly logger: ILogger, + private readonly validator: Validator, + private readonly importKeystoresPath: string + ) {} + + getKeystorePathInfoForKey = (pubkey: string): {keystoreFilePath: string; lockFilePath: string} => { + const keystoreFilename = `${KEY_IMPORTED_PREFIX}_${pubkey}.json`; + const keystoreFilePath = path.join(this.importKeystoresPath, keystoreFilename); + return { + keystoreFilePath, + lockFilePath: `${keystoreFilePath}${LOCK_FILE_EXT}`, + }; + }; + + /** + * List all validating pubkeys known to and decrypted by this keymanager binary + * + * https://github.com/ethereum/keymanager-APIs/blob/0c975dae2ac6053c8245ebdb6a9f27c2f114f407/keymanager-oapi.yaml + */ + async listKeys(): Promise<{ + data: { + validatingPubkey: PubkeyHex; + /** The derivation path (if present in the imported keystore) */ + derivationPath?: string; + /** The key associated with this pubkey cannot be deleted from the API */ + readonly?: boolean; + }[]; + }> { + const pubkeys = this.validator.validatorStore.votingPubkeys(); + return { + data: pubkeys.map((pubkey) => ({ + validatingPubkey: pubkey, + derivationPath: "", + readonly: this.validator.validatorStore.getSigner(pubkey)?.type !== SignerType.Local, + })), + }; + } + + /** + * Import keystores generated by the Eth2.0 deposit CLI tooling. `passwords[i]` must unlock `keystores[i]`. + * + * Users SHOULD send slashing_protection data associated with the imported pubkeys. MUST follow the format defined in + * EIP-3076: Slashing Protection Interchange Format. + * + * @param keystores JSON-encoded keystore files generated with the Launchpad + * @param passwords Passwords to unlock imported keystore files. `passwords[i]` must unlock `keystores[i]` + * @param slashingProtection Slashing protection data for some of the keys of `keystores` + * @returns Status result of each `request.keystores` with same length and order of `request.keystores` + * + * https://github.com/ethereum/keymanager-APIs/blob/0c975dae2ac6053c8245ebdb6a9f27c2f114f407/keymanager-oapi.yaml + */ + async importKeystores( + keystoresStr: KeystoreStr[], + passwords: string[], + slashingProtectionStr: SlashingProtectionData + ): Promise<{ + data: { + status: ImportStatus; + message?: string; + }[]; + }> { + const interchange = (slashingProtectionStr as unknown) as Interchange; + await this.validator.validatorStore.importInterchange(interchange); + + const statuses: {status: ImportStatus; message?: string}[] = []; + + for (let i = 0; i < keystoresStr.length; i++) { + try { + const keystoreStr = keystoresStr[i]; + const password = passwords[i]; + if (password === undefined) { + throw Error(`No password for keystores[${i}]`); + } + + const keystore = Keystore.parse(keystoreStr); + + // Check for duplicates and skip keystore before decrypting + if (this.validator.validatorStore.hasVotingPubkey(keystore.pubkey)) { + statuses[i] = {status: ImportStatus.duplicate}; + continue; + } + + const secretKey = SecretKey.fromBytes(await keystore.decrypt(password)); + const pubKey = secretKey.toPublicKey().toHex(); + this.validator.validatorStore.addSigner({type: SignerType.Local, secretKey}); + + const keystorePathInfo = this.getKeystorePathInfoForKey(pubKey); + + // Persist keys for latter restarts + await fs.promises.writeFile(keystorePathInfo.keystoreFilePath, keystoreStr, {encoding: "utf8"}); + const lockFile = getLockFile(); + lockFile.lockSync(keystorePathInfo.lockFilePath); + + statuses[i] = {status: ImportStatus.imported}; + } catch (e) { + statuses[i] = {status: ImportStatus.error, message: (e as Error).message}; + } + } + + return {data: statuses}; + } + + /** + * DELETE must delete all keys from `request.pubkeys` that are known to the keymanager and exist in its + * persistent storage. Additionally, DELETE must fetch the slashing protection data for the requested keys from + * persistent storage, which must be retained (and not deleted) after the response has been sent. Therefore in the + * case of two identical delete requests being made, both will have access to slashing protection data. + * + * In a single atomic sequential operation the keymanager must: + * 1. Guarantee that key(s) can not produce any more signature; only then + * 2. Delete key(s) and serialize its associated slashing protection data + * + * DELETE should never return a 404 response, even if all pubkeys from request.pubkeys have no extant keystores + * nor slashing protection data. + * + * Slashing protection data must only be returned for keys from `request.pubkeys` for which a + * `deleted` or `not_active` status is returned. + * + * @param pubkeys List of public keys to delete. + * @returns Deletion status of all keys in `request.pubkeys` in the same order. + * + * https://github.com/ethereum/keymanager-APIs/blob/0c975dae2ac6053c8245ebdb6a9f27c2f114f407/keymanager-oapi.yaml + */ + async deleteKeystores( + pubkeysHex: string[] + ): Promise<{ + data: { + status: DeletionStatus; + message?: string; + }[]; + slashingProtection: SlashingProtectionData; + }> { + const deletedKey: boolean[] = []; + const statuses = new Array<{status: DeletionStatus; message?: string}>(pubkeysHex.length); + + for (let i = 0; i < pubkeysHex.length; i++) { + try { + const pubkeyHex = pubkeysHex[i]; + + // Skip unknown keys or remote signers + const signer = this.validator.validatorStore.getSigner(pubkeyHex); + if (!signer || signer?.type === SignerType.Remote) { + continue; + } + + // Remove key from live local signer + deletedKey[i] = signer?.type === SignerType.Local && this.validator.validatorStore.removeSigner(pubkeyHex); + + // Remove key from blockduties + // Remove from attestation duties + // Remove from Sync committee duties + // Remove from indices + this.validator.removeDutiesForKey(pubkeyHex); + const keystorePathInfo = this.getKeystorePathInfoForKey(pubkeyHex); + // Remove key from persistent storage + for (const keystoreFile of await fs.promises.readdir(this.importKeystoresPath)) { + if (keystoreFile.indexOf(pubkeyHex) !== -1) { + await fs.promises.unlink(keystorePathInfo.keystoreFilePath); + await fs.promises.unlink(keystorePathInfo.lockFilePath); + } + } + } catch (e) { + statuses[i] = {status: DeletionStatus.error, message: (e as Error).message}; + } + } + + const pubkeysBytes = pubkeysHex.map((pubkeyHex) => fromHexString(pubkeyHex)); + + const interchangeV5 = await this.validator.validatorStore.exportInterchange(pubkeysBytes, { + version: "5", + }); + + // After exporting slashing protection data in bulk, render the status + const pubkeysWithSlashingProtectionData = new Set(interchangeV5.data.map((data) => data.pubkey)); + for (let i = 0; i < pubkeysHex.length; i++) { + if (statuses[i]?.status === DeletionStatus.error) { + continue; + } + const status = deletedKey[i] + ? DeletionStatus.deleted + : pubkeysWithSlashingProtectionData.has(pubkeysHex[i]) + ? DeletionStatus.not_active + : DeletionStatus.not_found; + statuses[i] = {status}; + } + + return { + data: statuses, + slashingProtection: JSON.stringify(interchangeV5), + }; + } +} diff --git a/packages/keymanager-server/src/index.ts b/packages/keymanager-server/src/index.ts new file mode 100644 index 00000000000..f9f0166dec4 --- /dev/null +++ b/packages/keymanager-server/src/index.ts @@ -0,0 +1,3 @@ +export * from "./impl"; +export * from "./server"; +export * from "./util/lockfile"; diff --git a/packages/keymanager-server/src/server.ts b/packages/keymanager-server/src/server.ts new file mode 100644 index 00000000000..1fa2108fb23 --- /dev/null +++ b/packages/keymanager-server/src/server.ts @@ -0,0 +1,159 @@ +import fastify, {FastifyError, FastifyInstance} from "fastify"; +import fastifyCors from "fastify-cors"; +import bearerAuthPlugin from "fastify-bearer-auth"; +import querystring from "querystring"; +import {IncomingMessage} from "node:http"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import {toHexString} from "@chainsafe/ssz"; +export {allNamespaces} from "@chainsafe/lodestar-api"; +import {Api} from "@chainsafe/lodestar-api/keymanager"; +import {getRoutes} from "@chainsafe/lodestar-api/keymanager_server"; +import {registerRoutesGroup, RouteConfig} from "@chainsafe/lodestar-api/server"; +import {ErrorAborted, ILogger} from "@chainsafe/lodestar-utils"; +import {IChainForkConfig} from "@chainsafe/lodestar-config"; +import {join} from "node:path"; + +export type RestApiOptions = { + host: string; + cors: string; + port: number; + isAuthEnabled: boolean; + tokenDir?: string; +}; + +export const restApiOptionsDefault: RestApiOptions = { + host: "127.0.0.1", + port: 5062, + cors: "*", + isAuthEnabled: true, +}; + +export interface IRestApiModules { + config: IChainForkConfig; + logger: ILogger; + api: Api; +} + +const apiTokenFileName = "api-token.txt"; + +export class KeymanagerServer { + private readonly opts: RestApiOptions; + private readonly server: FastifyInstance; + private readonly logger: ILogger; + private readonly activeRequests = new Set(); + private readonly apiTokenPath: string | undefined; + private readonly bearerToken: string | undefined; + + constructor(optsArg: Partial, modules: IRestApiModules) { + this.logger = modules.logger; + // Apply opts defaults + const opts = { + ...restApiOptionsDefault, + ...Object.fromEntries(Object.entries(optsArg).filter(([_, v]) => v != null)), + }; + + if (opts.isAuthEnabled && opts.tokenDir) { + this.apiTokenPath = join(opts.tokenDir, apiTokenFileName); + // Generate a new token if token file does not exist or file do exist, but is empty + if (!fs.existsSync(this.apiTokenPath) || fs.readFileSync(this.apiTokenPath, "utf8").trim().length === 0) { + this.bearerToken = `api-token-${toHexString(crypto.randomBytes(32))}`; + fs.writeFileSync(this.apiTokenPath, this.bearerToken, {encoding: "utf8"}); + } else { + this.bearerToken = fs.readFileSync(this.apiTokenPath, "utf8").trim(); + } + } else { + this.logger.warn("Keymanager server started without authentication"); + } + + const server = fastify({ + logger: false, + ajv: {customOptions: {coerceTypes: "array"}}, + querystringParser: querystring.parse, + }); + + // Instantiate and register the keymanager routes + const routes = getRoutes(modules.config, modules.api); + registerRoutesGroup(server, routes); + + // To parse our ApiError -> statusCode + server.setErrorHandler((err, req, res) => { + if ((err as FastifyError).validation) { + void res.status(400).send((err as FastifyError).validation); + } else { + void res.status(500).send(err); + } + }); + + if (opts.cors) { + void server.register(fastifyCors, {origin: opts.cors}); + } + + if (opts.isAuthEnabled && this.bearerToken) { + void server.register(bearerAuthPlugin, {keys: new Set([this.bearerToken])}); + } + + // Log all incoming request to debug (before parsing). TODO: Should we hook latter in the lifecycle? https://www.fastify.io/docs/latest/Lifecycle/ + // Note: Must be an async method so fastify can continue the release lifecycle. Otherwise we must call done() or the request stalls + server.addHook("onRequest", async (req) => { + this.activeRequests.add(req.raw); + const url = req.raw.url ? req.raw.url.split("?")[0] : "-"; + this.logger.debug(`Req ${req.id} ${req.ip} ${req.raw.method}:${url}`); + }); + + // Log after response + server.addHook("onResponse", async (req, res) => { + this.activeRequests.delete(req.raw); + const {operationId} = res.context.config as RouteConfig; + this.logger.debug(`Res ${req.id} ${operationId} - ${res.raw.statusCode}`); + }); + + server.addHook("onError", async (req, res, err) => { + this.activeRequests.delete(req.raw); + // Don't log ErrorAborted errors, they happen on node shutdown and are not usefull + if (err instanceof ErrorAborted) return; + + const {operationId} = res.context.config as RouteConfig; + this.logger.error(`Req ${req.id} ${operationId} error`, {}, err); + }); + + this.opts = opts; + this.server = server; + } + + /** + * Start the REST API server. + */ + async listen(): Promise { + try { + const address = await this.server.listen(this.opts.port, this.opts.host); + this.logger.info("Started keymanager api server", {address}); + if (this.apiTokenPath) { + this.logger.info("Keymanager bearer access token located at:", this.apiTokenPath); + } + } catch (e) { + this.logger.error( + "Error starting Keymanager api server", + {host: this.opts.host, port: this.opts.port}, + e as Error + ); + throw e; + } + } + + /** + * Close the server instance and terminate all existing connections. + */ + async close(): Promise { + // In NodeJS land calling close() only causes new connections to be rejected. + // Existing connections can prevent .close() from resolving for potentially forever. + // In Lodestar case when the BeaconNode wants to close we will just abruptly terminate + // all existing connections for a fast shutdown. + // Inspired by https://github.com/gajus/http-terminator/ + for (const req of this.activeRequests) { + req.destroy(Error("Closing")); + } + + await this.server.close(); + } +} diff --git a/packages/cli/src/util/lockfile.ts b/packages/keymanager-server/src/util/lockfile.ts similarity index 63% rename from packages/cli/src/util/lockfile.ts rename to packages/keymanager-server/src/util/lockfile.ts index 2306a81d2a6..bf8839912bc 100644 --- a/packages/cli/src/util/lockfile.ts +++ b/packages/keymanager-server/src/util/lockfile.ts @@ -4,14 +4,16 @@ type Lockfile = { }; let lockFile: Lockfile | null = null; - +export const LOCK_FILE_EXT = ".lock"; /** * When lockfile it's required it registers listeners to process * Since it's only used by the validator client, require lazily to not pollute * beacon_node client context */ export function getLockFile(): Lockfile { - // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports - if (!lockFile) lockFile = require("lockfile") as Lockfile; + if (!lockFile) { + // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports + lockFile = require("lockfile") as Lockfile; + } return lockFile; } diff --git a/packages/keymanager-server/test/setup.ts b/packages/keymanager-server/test/setup.ts new file mode 100644 index 00000000000..bacbbf65f91 --- /dev/null +++ b/packages/keymanager-server/test/setup.ts @@ -0,0 +1,9 @@ +import {init} from "@chainsafe/bls"; + +// blst-native initialization is syncronous +// Initialize bls here instead of in before() so it's available inside describe() blocks +init("blst-native").catch((e: Error) => { + // eslint-disable-next-line no-console + console.error(e); + process.exit(1); +}); diff --git a/packages/keymanager-server/tsconfig.build.json b/packages/keymanager-server/tsconfig.build.json new file mode 100644 index 00000000000..49aab410147 --- /dev/null +++ b/packages/keymanager-server/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.build.json", + "include": ["src"], + "compilerOptions": { + "outDir": "lib", + "typeRoots": ["../../node_modules/@types", "./node_modules/@types"] + } +} diff --git a/packages/keymanager-server/tsconfig.json b/packages/keymanager-server/tsconfig.json new file mode 100644 index 00000000000..c5f850d41f4 --- /dev/null +++ b/packages/keymanager-server/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "typeRoots": ["../../node_modules/@types", "./node_modules/@types"] + } +} diff --git a/packages/lodestar/test/e2e/chain/lightclient.test.ts b/packages/lodestar/test/e2e/chain/lightclient.test.ts index a63116de0db..dc18fd6e32e 100644 --- a/packages/lodestar/test/e2e/chain/lightclient.test.ts +++ b/packages/lodestar/test/e2e/chain/lightclient.test.ts @@ -37,6 +37,14 @@ describe("chain / lightclient", function () { // This is a rare event, with maxLcHeadTrackingDiffSlots = 4, SECONDS_PER_SLOT = 1 this.retries(2); + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + it("Lightclient track head on server configuration", async function () { this.timeout("10 min"); @@ -67,6 +75,11 @@ describe("chain / lightclient", function () { genesisTime, logger: loggerNodeA, }); + + afterEachCallbacks.push(async () => { + await bn.close(); + }); + const {validators} = await getAndInitDevValidators({ node: bn, validatorsPerClient: validatorCount, @@ -76,6 +89,10 @@ describe("chain / lightclient", function () { testLoggerOpts: {...testLoggerOpts, logLevel: LogLevel.error}, }); + afterEachCallbacks.push(async () => { + await Promise.all(validators.map((v) => v.stop())); + }); + await Promise.all(validators.map((validator) => validator.start())); // This promise chain does: @@ -107,6 +124,10 @@ describe("chain / lightclient", function () { checkpointRoot: fromHexString(head.blockRoot), }); + afterEachCallbacks.push(async () => { + lightclient.stop(); + }); + loggerLC.important("Initialized lightclient", {headSlot: lightclient.getHead().slot}); lightclient.start(); @@ -158,8 +179,5 @@ describe("chain / lightclient", function () { const headSummary = bn.chain.forkChoice.getHead(); const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); if (!head) throw Error("First beacon node has no head block"); - - await Promise.all(validators.map((v) => v.stop())); - await bn.close(); }); }); diff --git a/packages/lodestar/test/e2e/keymanager/keymanager.test.ts b/packages/lodestar/test/e2e/keymanager/keymanager.test.ts new file mode 100644 index 00000000000..fa976e1fa70 --- /dev/null +++ b/packages/lodestar/test/e2e/keymanager/keymanager.test.ts @@ -0,0 +1,512 @@ +import chaiAsPromised from "chai-as-promised"; +import chai, {expect} from "chai"; +import fs from "node:fs"; +import path from "node:path"; +import tmp from "tmp"; +import {createIBeaconConfig, IBeaconConfig, IChainConfig} from "@chainsafe/lodestar-config"; +import {KeymanagerApi, KeymanagerServer} from "@chainsafe/lodestar-keymanager-server"; +import {chainConfig as chainConfigDef} from "@chainsafe/lodestar-config/default"; +import {HttpClient} from "@chainsafe/lodestar-api/src"; +import {getClient} from "@chainsafe/lodestar-api/src/keymanager/client"; +import {ISlashingProtection, Validator} from "@chainsafe/lodestar-validator"; +import {ByteVector, fromHexString} from "@chainsafe/ssz"; +import {WinstonLogger} from "@chainsafe/lodestar-utils"; +import {ssz} from "@chainsafe/lodestar-types"; +import {LogLevel, testLogger, TestLoggerOpts} from "../../utils/logger"; +import {getDevBeaconNode} from "../../utils/node/beacon"; +import {getAndInitDevValidators, getAndInitValidatorsWithKeystore} from "../../utils/node/validator"; +import {getKeystoreForPubKey1, getKeystoreForPubKey2} from "../../utils/node/keymanager"; +import {logFilesDir} from "../../sim/params"; +/* eslint-disable @typescript-eslint/naming-convention */ + +chai.use(chaiAsPromised); + +describe("keymanager delete and import test", async function () { + const validatorCount = 1; + const SECONDS_PER_SLOT = 2; + const ALTAIR_FORK_EPOCH = 0; + const key1 = "0x97b1b00d3c1888b5715c2c88bf1df7b0ad715388079de211bdc153697b69b868c671af3b2d86c5cdfbade48d03888ab4"; + const key2 = "0xa74e11fd129b9bafc2d6afad4944cd289c238139130a7abafe7b28dde1923a0e4833ad776f9e0d7aaaecd9f0acbfedd3"; + const beaconParams: Partial = { + SECONDS_PER_SLOT: SECONDS_PER_SLOT, + }; + + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + + it("should migrate validator from one VC to another", async function () { + this.timeout("10 min"); + + const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; + const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); + const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); + + const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: beaconParams, + options: {sync: {isSingleNode: true}}, + validatorCount, + logger: loggerNodeA, + }); + + afterEachCallbacks.push(() => bn.close()); + + const vc1Info = await getAndInitValidatorsWithKeystore({ + node: bn, + keystoreContent: getKeystoreForPubKey1(), + keystorePubKey: key1, + useRestApi: false, + testLoggerOpts, + }); + + afterEachCallbacks.push(() => vc1Info.validator.stop()); + + const vc2Info = await getAndInitValidatorsWithKeystore({ + node: bn, + keystoreContent: getKeystoreForPubKey2(), + keystorePubKey: key2, + useRestApi: false, + testLoggerOpts, + }); + + afterEachCallbacks.push(() => vc2Info.validator.stop()); + + const portKM1 = 10000; + const portKM2 = 10001; + + const keymanagerServerForVC1 = createKeymanager( + vc1Info.validator, + vc1Info.slashingProtection, + vc1Info.tempDirs.keystoreDir.name, + portKM1, + config, + loggerNodeA + ); + + afterEachCallbacks.push(() => keymanagerServerForVC1.close()); + + const keymanagerServerForVC2 = createKeymanager( + vc2Info.validator, + vc2Info.slashingProtection, + vc2Info.tempDirs.keystoreDir.name, + portKM2, + config, + loggerNodeA + ); + + afterEachCallbacks.push(() => keymanagerServerForVC1.close()); + // Register clean up + // eslint-disable-next-line @typescript-eslint/no-unsafe-call,@typescript-eslint/no-unsafe-member-access + afterEachCallbacks.push(async () => { + await Promise.all([vc1Info.validator.stop(), vc2Info.validator.stop()]); + vc1Info.tempDirs.keystoreDir.removeCallback(); + vc1Info.tempDirs.passwordFile.removeCallback(); + vc2Info.tempDirs.keystoreDir.removeCallback(); + vc2Info.tempDirs.passwordFile.removeCallback(); + }); + + await keymanagerServerForVC1.listen(); + await keymanagerServerForVC2.listen(); + + // 1. CONFIRM KEYS BEFORE DELETION AND IMPORT + const clientKM1 = getClient(config, new HttpClient({baseUrl: `http://127.0.0.1:${portKM1}`})); + const clientKM2 = getClient(config, new HttpClient({baseUrl: `http://127.0.0.1:${portKM2}`})); + + // 1.a. CONFIRM PRESENCE KEYS VIA API + + // confirm pubkey key1 in first validator client + let km1ListKeyResult = await clientKM1.listKeys(); + expect(km1ListKeyResult.data.map((d) => d.validatingPubkey)).to.deep.equal( + [key1], + "confirm pubkey key1 in first validator client" + ); + + // confirm pubkey key2 in second validator client + let km2ListKeyResult = await clientKM2.listKeys(); + expect(km2ListKeyResult.data.map((d) => d.validatingPubkey)).to.deep.equal( + [key2], + "confirm pubkey key2 in second validator client" + ); + + // 1.b. CONFIRM PRESENCE OF KEYS VIA FILE SYSTEM + + // confirm keystore for k1 exist on file + expect( + dirContainFileWithPubkeyInFilename(vc1Info.tempDirs.keystoreDir.name, [key1]), + "key1 should exist on file for vc1" + ).to.be.true; + expect( + dirContainFileWithPubkeyInFilename(vc1Info.tempDirs.keystoreDir.name, [key2]), + "key2 should not exist on file for vc1" + ).to.be.false; + // confirm keystore for k2 exist on file + expect( + dirContainFileWithPubkeyInFilename(vc2Info.tempDirs.keystoreDir.name, [key2]), + "key2 should exist on file for vc2" + ).to.be.true; + expect( + dirContainFileWithPubkeyInFilename(vc2Info.tempDirs.keystoreDir.name, [key1]), + "Key1 should not exist on file for vc2" + ).to.be.false; + + // 2. DELETE PUBKEY K1 from first validator client + // delete pubkey key1 in vc1Info + const km1DeleteKeyResult = await clientKM1.deleteKeystores([key1]); + + // 2.a CONFIRM DELETION OF K1 from first validator client USING API AND FILESYSTEM + // confirm pubkey key1 is no longer in vc1Info + km1ListKeyResult = await clientKM1.listKeys(); + expect(km1ListKeyResult.data.length).to.equal(0, "key1 is no longer in vc1"); + // confirm keystore for k1 no longer exist on file + expect( + dirContainFileWithPubkeyInFilename(vc1Info.tempDirs.keystoreDir.name, [key1]), + "keystore for k1 no longer exist on file for vc1" + ).to.be.false; + + // 3. IMPORT PUBKEY K1 to SECOND VALIDATOR CLIENT + + expect( + vc2Info.validator.validatorStore.signAttestation( + createAttesterDuty(fromHexString(key1), 0, 0, 1), + ssz.phase0.AttestationData.defaultValue(), + 1 + ), + "3.a Confirmation before import, that signing with k1 on vc2 throws" + ).to.eventually.throw; + + // Import k1 to vc2 + const importResult = await clientKM2.importKeystores( + [vc1Info.keystoreContent], + ["test123!"], + JSON.parse(km1DeleteKeyResult.slashingProtection) + ); + + // 3.b. COMFIRM IMPORT RESPONSE + expect(importResult.data[0].status).to.be.equal("imported", "3.b. confirm import response"); + + // 4 CONFIRM PRESENCE OF IMPORTED KEY IN SECOND VALIDATOR CLIENT. + + km2ListKeyResult = await clientKM2.listKeys(); + expect(km2ListKeyResult.data.map((d) => d.validatingPubkey)).to.deep.equal( + [key2, key1], + "4.a confirm imported keys in vc 2" + ); + + expect( + dirContainFileWithPubkeyInFilename(vc2Info.tempDirs.keystoreDir.name, [key1, key2]), + "4.b Confirm imported and previous key still exist via file system" + ).to.be.true; + + expect( + vc1Info.validator.validatorStore.signAttestation( + createAttesterDuty(fromHexString(key1), 0, 0, 1), + ssz.phase0.AttestationData.defaultValue(), + 1 + ), + "4.c Confirm vc1 cannot sign with k1" + ).to.eventually.throw; + + expect( + vc2Info.validator.validatorStore.signAttestation( + createAttesterDuty(fromHexString(key1), 0, 0, 1), + ssz.phase0.AttestationData.defaultValue(), + 1 + ), + "4.d Confirm vc2 can now sign with k1" + ).to.eventually.not.throw; + }); + + it("should not delete external signers", async function () { + this.timeout("10 min"); + + const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; + const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); + const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); + + const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: beaconParams, + options: {sync: {isSingleNode: true}}, + validatorCount, + logger: loggerNodeA, + }); + + afterEachCallbacks.push(() => bn.close()); + + const externalSignerPort = 38000; + const externalSignerUrl = `http://localhost:${externalSignerPort}`; + + const {validators, secretKeys} = await getAndInitDevValidators({ + node: bn, + validatorsPerClient: 1, + validatorClientCount: 1, + startIndex: 0, + // At least one sim test must use the REST API for beacon <-> validator comms + useRestApi: true, + testLoggerOpts, + externalSignerUrl: externalSignerUrl, + }); + + afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.stop()))); + + const keymanagerApi = new KeymanagerApi(loggerNodeA, validators[0], "/test/path"); + + const kmPort = 10003; + + const keymanagerServer = new KeymanagerServer( + {host: "127.0.0.1", port: kmPort, cors: "*", isAuthEnabled: false, tokenDir: logFilesDir}, + {config, logger: loggerNodeA, api: keymanagerApi} + ); + + afterEachCallbacks.push(() => keymanagerServer.close()); + + await keymanagerServer.listen(); + + const client = getClient(config, new HttpClient({baseUrl: `http://127.0.0.1:${kmPort}`})); + + expect((await client.listKeys()).data).to.be.deep.equal( + [ + { + validatingPubkey: `${secretKeys[0].toPublicKey().toHex()}`, + derivationPath: "", + readonly: true, + }, + ], + "listKeys should return key that is readonly" + ); + + expect((await client.deleteKeystores([key1])).data).to.deep.equal( + [{status: "not_active"}], + "deleteKeystores should not delete readonly key" + ); + }); + + describe("Authentication tests", async function () { + it("should deny request if authentication is on and no bearer token is provided", async function () { + this.timeout("10 min"); + + const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; + const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); + const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); + + const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: beaconParams, + options: {sync: {isSingleNode: true}}, + validatorCount, + logger: loggerNodeA, + }); + + afterEachCallbacks.push(() => bn.close()); + + const {validators, secretKeys: _secretKeys} = await getAndInitDevValidators({ + node: bn, + validatorsPerClient: validatorCount, + validatorClientCount: 1, + startIndex: 0, + useRestApi: false, + testLoggerOpts, + }); + + afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.stop()))); + + const keymanagerApi = new KeymanagerApi(loggerNodeA, validators[0], "/test/path"); + + const kmPort = 10003; + + const tokenDir = tmp.dirSync({unsafeCleanup: true}); + afterEachCallbacks.push(() => tokenDir.removeCallback()); + + // by default auth is on + const keymanagerServer = new KeymanagerServer( + {host: "127.0.0.1", port: kmPort, cors: "*", tokenDir: tokenDir.name}, + {config, logger: loggerNodeA, api: keymanagerApi} + ); + + afterEachCallbacks.push(() => keymanagerServer.close()); + + await keymanagerServer.listen(); + + const client = getClient(config, new HttpClient({baseUrl: `http://127.0.0.1:${kmPort}`})); + + // Listing keys is denied + try { + await client.listKeys(); + } catch (e) { + // prettier-ignore + expect((e as Error).message).to.equal("Unauthorized: {\"error\":\"missing authorization header\"}", "Expect list request to be denied"); + } + + // Deleting keys is denied + try { + await client.deleteKeystores([key1]); + } catch (e) { + // prettier-ignore + expect((e as Error).message).to.equal("Unauthorized: {\"error\":\"missing authorization header\"}", "Expect delete request to be denied"); + } + + // importing keys is denied + try { + await client.importKeystores(["some keystore string"], ["some password"], "some slashing protecting)"); + } catch (e) { + // prettier-ignore + expect((e as Error).message).to.equal("Unauthorized: {\"error\":\"missing authorization header\"}", "Expect import request to be denied"); + } + }); + + it("should generate bearer token if auth is on and no bearer token file exist", async function () { + this.timeout("10 min"); + + const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; + const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); + const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); + + const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: beaconParams, + options: {sync: {isSingleNode: true}}, + validatorCount, + logger: loggerNodeA, + }); + + afterEachCallbacks.push(() => bn.close()); + + const {validators, secretKeys: _secretKeys} = await getAndInitDevValidators({ + node: bn, + validatorsPerClient: validatorCount, + validatorClientCount: 1, + startIndex: 0, + useRestApi: false, + testLoggerOpts, + }); + + afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.stop()))); + + const keymanagerApi = new KeymanagerApi(loggerNodeA, validators[0], "/test/path"); + + const kmPort = 10003; + + const tokenDir = tmp.dirSync({unsafeCleanup: true}); + afterEachCallbacks.push(() => tokenDir.removeCallback()); + + expect(() => { + fs.readFileSync(path.join(tokenDir.name, "api-token.txt")); + }, "api.token should not be present before keymanager server is started").to.throw(); + + // by default auth is on + new KeymanagerServer( + {host: "127.0.0.1", port: kmPort, cors: "*", tokenDir: tokenDir.name}, + {config, logger: loggerNodeA, api: keymanagerApi} + ); + + expect( + fs.readFileSync(path.join(tokenDir.name, "api-token.txt")), + "api.token should be present and not be empty after keymanager server is started" + ).to.not.be.undefined; + }); + + it("should generate bearer token if auth is on and empty bearer token file exist", async function () { + this.timeout("10 min"); + + const chainConfig: IChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; + const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); + const config = createIBeaconConfig(chainConfig, genesisValidatorsRoot); + + const testLoggerOpts: TestLoggerOpts = {logLevel: LogLevel.info}; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: beaconParams, + options: {sync: {isSingleNode: true}}, + validatorCount, + logger: loggerNodeA, + }); + + afterEachCallbacks.push(() => bn.close()); + + const {validators, secretKeys: _secretKeys} = await getAndInitDevValidators({ + node: bn, + validatorsPerClient: validatorCount, + validatorClientCount: 1, + startIndex: 0, + useRestApi: false, + testLoggerOpts, + }); + + afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.stop()))); + + const keymanagerApi = new KeymanagerApi(loggerNodeA, validators[0], "/test/path"); + + const kmPort = 10003; + + const tokenDir = tmp.dirSync({unsafeCleanup: true}); + afterEachCallbacks.push(() => tokenDir.removeCallback()); + + // create an empty api-token.txt + fs.closeSync(fs.openSync(path.join(tokenDir.name, "api-token.txt"), "w")); + expect(fs.readFileSync(path.join(tokenDir.name, "api-token.txt")).length).to.equal( + 0, + "api.token.txt should be empty before keymanager is started" + ); + + // by default auth is on + new KeymanagerServer( + {host: "127.0.0.1", port: kmPort, cors: "*", tokenDir: tokenDir.name}, + {config, logger: loggerNodeA, api: keymanagerApi} + ); + + expect(fs.readFileSync(path.join(tokenDir.name, "api-token.txt")).length).to.be.greaterThan( + 0, + "api.token.txt should not be empty after keymanager is started" + ); + }); + }); +}); + +function dirContainFileWithPubkeyInFilename(dir: string, pubkeys: string[]): boolean { + return fs.readdirSync(dir).some((name) => { + return pubkeys.some((pubkey) => name.indexOf(pubkey) !== -1); + }); +} + +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +function createAttesterDuty(pubkey: ByteVector, slot: number, committeeIndex: number, validatorIndex: number) { + return { + slot: slot, + committeeIndex: committeeIndex, + committeeLength: 120, + committeesAtSlot: 120, + validatorCommitteeIndex: 1, + validatorIndex: validatorIndex, + pubkey: pubkey, + }; +} + +function createKeymanager( + vc: Validator, + slashingProtection: ISlashingProtection, + importKeystoresPath: string, + port: number, + config: IBeaconConfig, + logger: WinstonLogger +): KeymanagerServer { + const keymanagerApi = new KeymanagerApi(logger, vc, importKeystoresPath); + + return new KeymanagerServer( + {host: "127.0.0.1", port, cors: "*", isAuthEnabled: false, tokenDir: logFilesDir}, + {config, logger: logger, api: keymanagerApi} + ); +} diff --git a/packages/lodestar/test/e2e/sync/finalizedSync.test.ts b/packages/lodestar/test/e2e/sync/finalizedSync.test.ts index 63944006dff..a76e8d1ddff 100644 --- a/packages/lodestar/test/e2e/sync/finalizedSync.test.ts +++ b/packages/lodestar/test/e2e/sync/finalizedSync.test.ts @@ -17,6 +17,14 @@ describe("sync / finalized sync", function () { SECONDS_PER_SLOT: 2, }; + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + it("should do a finalized sync from another BN", async function () { this.timeout("10 min"); @@ -30,6 +38,9 @@ describe("sync / finalized sync", function () { validatorCount, logger: loggerNodeA, }); + + afterEachCallbacks.push(() => bn.close()); + const {validators} = await getAndInitDevValidators({ node: bn, validatorsPerClient: validatorCount, @@ -39,6 +50,8 @@ describe("sync / finalized sync", function () { testLoggerOpts, }); + afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.stop()))); + await Promise.all(validators.map((validator) => validator.start())); await waitForEvent(bn.chain.emitter, ChainEvent.finalized, 240000); @@ -52,6 +65,8 @@ describe("sync / finalized sync", function () { logger: loggerNodeB, }); + afterEachCallbacks.push(() => bn2.close()); + const headSummary = bn.chain.forkChoice.getHead(); const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); if (!head) throw Error("First beacon node has no head block"); @@ -66,8 +81,5 @@ describe("sync / finalized sync", function () { } catch (e) { assert.fail("Failed to sync to other node in time"); } - await bn2.close(); - await Promise.all(validators.map((v) => v.stop())); - await bn.close(); }); }); diff --git a/packages/lodestar/test/e2e/sync/unknownBlockSync.test.ts b/packages/lodestar/test/e2e/sync/unknownBlockSync.test.ts index d75eb0b6d04..24f427143bc 100644 --- a/packages/lodestar/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/lodestar/test/e2e/sync/unknownBlockSync.test.ts @@ -19,6 +19,14 @@ describe("sync / unknown block sync", function () { SECONDS_PER_SLOT: 2, }; + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + it("should do an unknown block sync from another BN", async function () { this.timeout("10 min"); @@ -42,6 +50,9 @@ describe("sync / unknown block sync", function () { validatorCount, logger: loggerNodeA, }); + + afterEachCallbacks.push(() => bn.close()); + const {validators} = await getAndInitDevValidators({ node: bn, validatorsPerClient: validatorCount, @@ -51,6 +62,8 @@ describe("sync / unknown block sync", function () { testLoggerOpts, }); + afterEachCallbacks.push(() => Promise.all(validators.map((v) => v.stop()))); + await Promise.all(validators.map((validator) => validator.start())); await waitForEvent(bn.chain.emitter, ChainEvent.checkpoint, 240000); @@ -64,6 +77,8 @@ describe("sync / unknown block sync", function () { logger: loggerNodeB, }); + afterEachCallbacks.push(() => bn2.close()); + const headSummary = bn.chain.forkChoice.getHead(); const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); if (!head) throw Error("First beacon node has no head block"); @@ -83,9 +98,5 @@ describe("sync / unknown block sync", function () { // Wait for NODE-A head to be processed in NODE-B without range sync await waitForSynced; - - await bn2.close(); - await Promise.all(validators.map((v) => v.stop())); - await bn.close(); }); }); diff --git a/packages/lodestar/test/e2e/sync/wss.test.ts b/packages/lodestar/test/e2e/sync/wss.test.ts index ab4f239d145..80bed2a8fb3 100644 --- a/packages/lodestar/test/e2e/sync/wss.test.ts +++ b/packages/lodestar/test/e2e/sync/wss.test.ts @@ -101,7 +101,7 @@ describe("Start from WSS", function () { throw e; } - const weakSubjectivityServerUrl = "http://127.0.0.1:9596"; + const weakSubjectivityServerUrl = "http://127.0.0.1:19596"; loggerNodeB.important("Fetching weak subjectivity state ", {weakSubjectivityServerUrl}); const {wsState, wsCheckpoint} = await fetchWeakSubjectivityState(config, {weakSubjectivityServerUrl}); loggerNodeB.important("Fetched wss state"); diff --git a/packages/lodestar/test/sim/multiNodeSingleThread.test.ts b/packages/lodestar/test/sim/multiNodeSingleThread.test.ts index 4fc66acaa55..d0675990794 100644 --- a/packages/lodestar/test/sim/multiNodeSingleThread.test.ts +++ b/packages/lodestar/test/sim/multiNodeSingleThread.test.ts @@ -35,7 +35,14 @@ describe("Run multi node single thread interop validators (no eth1) until checkp {nodeCount: 4, validatorsPerNode: 8, event: ChainEvent.justified, altairForkEpoch: 2}, ]; - let onDoneHandlers: (() => Promise)[] = []; + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach("Stop nodes and validators", async function () { + this.timeout("10 min"); + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); // TODO test multiNode with remote; @@ -79,6 +86,19 @@ describe("Run multi node single thread interop validators (no eth1) until checkp testLoggerOpts, }); + afterEachCallbacks.push(async () => { + await Promise.all(validators.map((validator) => validator.stop())); + console.log("--- Stopped all validators ---"); + // wait for 1 slot + await sleep(1 * testParams.SECONDS_PER_SLOT * 1000); + + stopInfoTracker(); + await Promise.all(nodes.map((node) => node.close())); + console.log("--- Stopped all nodes ---"); + // Wait a bit for nodes to shutdown + await sleep(3000); + }); + loggers.push(logger); nodes.push(node); validators.push(...nodeValidators); @@ -86,19 +106,6 @@ describe("Run multi node single thread interop validators (no eth1) until checkp const stopInfoTracker = simTestInfoTracker(nodes[0], loggers[0]); - onDoneHandlers.push(async () => { - await Promise.all(validators.map((validator) => validator.stop())); - console.log("--- Stopped all validators ---"); - // wait for 1 slot - await sleep(1 * testParams.SECONDS_PER_SLOT * 1000); - - stopInfoTracker(); - await Promise.all(nodes.map((node) => node.close())); - console.log("--- Stopped all nodes ---"); - // Wait a bit for nodes to shutdown - await sleep(3000); - }); - // Connect all nodes with each other for (let i = 0; i < nodeCount; i++) { for (let j = 0; j < nodeCount; j++) { @@ -116,12 +123,4 @@ describe("Run multi node single thread interop validators (no eth1) until checkp console.log("--- All nodes reached justified checkpoint ---"); }); } - - afterEach("Stop nodes and validators", async function () { - this.timeout(20000); - for (const onDoneHandler of onDoneHandlers) { - await onDoneHandler(); - } - onDoneHandlers = []; - }); }); diff --git a/packages/lodestar/test/utils/node/beacon.ts b/packages/lodestar/test/utils/node/beacon.ts index 946e108f176..61721557387 100644 --- a/packages/lodestar/test/utils/node/beacon.ts +++ b/packages/lodestar/test/utils/node/beacon.ts @@ -68,7 +68,7 @@ export async function getDevBeaconNode( { db: {name: tmpDir.name}, eth1: {enabled: false}, - api: {rest: {api: ["beacon", "config", "events", "node", "validator"]}}, + api: {rest: {api: ["beacon", "config", "events", "node", "validator"], port: 19596}}, metrics: {enabled: false}, network: {discv5: null}, } as Partial, diff --git a/packages/lodestar/test/utils/node/keymanager.ts b/packages/lodestar/test/utils/node/keymanager.ts new file mode 100644 index 00000000000..9e267f59ace --- /dev/null +++ b/packages/lodestar/test/utils/node/keymanager.ts @@ -0,0 +1,69 @@ +export function getKeystoreForPubKey1(): string { + return JSON.stringify({ + crypto: { + kdf: { + function: "scrypt", + params: { + dklen: 32, + n: 262144, + r: 8, + p: 1, + salt: "87f8e61bd461206ebbb222f2e789322504b6543067a8b49f2c29f35f203a56c5", + }, + message: "", + }, + checksum: { + function: "sha256", + params: {}, + message: "e3b7f6a0dc99543fa62afd4bcdf2a49b4ee8075609389eaa0bfeeb8987fcf8b8", + }, + cipher: { + function: "aes-128-ctr", + params: { + iv: "84008836292fbc9bd9efb50d95939cdc", + }, + message: "b81e6288a4307b8e29f2e952cecc5642e0832ae7123b93306702ec48cdf2f8d9", + }, + }, + description: "", + pubkey: "97b1b00d3c1888b5715c2c88bf1df7b0ad715388079de211bdc153697b69b868c671af3b2d86c5cdfbade48d03888ab4", + path: "m/12381/3600/0/0/0", + uuid: "537500a4-37ae-48f3-8ac2-2deda5285699", + version: 4, + }).trim(); +} + +export function getKeystoreForPubKey2(): string { + return JSON.stringify({ + crypto: { + kdf: { + function: "scrypt", + params: { + dklen: 32, + n: 262144, + r: 8, + p: 1, + salt: "6e179aeba4e5ac240b326cafe9a5de4ed7c17ac956b3c06537b384a508f5a818", + }, + message: "", + }, + checksum: { + function: "sha256", + params: {}, + message: "5436d7e035b60c08f9b285a5251ee5f5e2275e44ed161cba4352f1c1da869697", + }, + cipher: { + function: "aes-128-ctr", + params: { + iv: "7e87e2c3ede5e95aa86df569934b5e5c", + }, + message: "c06ebc0a02c61be5dafbe59e3d286e762f3b9fe0505176bd5504ed49ef90373a", + }, + }, + description: "", + pubkey: "a74e11fd129b9bafc2d6afad4944cd289c238139130a7abafe7b28dde1923a0e4833ad776f9e0d7aaaecd9f0acbfedd3", + path: "m/12381/3600/0/0/0", + uuid: "5c0169d3-c132-4581-8e7c-afcbf45000cf", + version: 4, + }).trim(); +} diff --git a/packages/lodestar/test/utils/node/validator.ts b/packages/lodestar/test/utils/node/validator.ts index 2150796b046..86776af8518 100644 --- a/packages/lodestar/test/utils/node/validator.ts +++ b/packages/lodestar/test/utils/node/validator.ts @@ -1,10 +1,111 @@ -import tmp from "tmp"; +import tmp, {DirResult, FileResult} from "tmp"; +import fs from "node:fs"; +import path from "node:path"; import {LevelDbController} from "@chainsafe/lodestar-db"; import {interopSecretKey} from "@chainsafe/lodestar-beacon-state-transition"; -import {SlashingProtection, Validator, Signer, SignerType} from "@chainsafe/lodestar-validator"; +import { + SlashingProtection, + Validator, + Signer, + SignerType, + ISlashingProtection, + SignerLocal, +} from "@chainsafe/lodestar-validator"; import {BeaconNode} from "../../../src/node"; import {testLogger, TestLoggerOpts} from "../logger"; import {SecretKey} from "@chainsafe/bls"; +import {getLocalSecretKeys} from "@chainsafe/lodestar-cli/src/cmds/validator/keys"; +import {IValidatorCliArgs} from "@chainsafe/lodestar-cli/src/cmds/validator/options"; +import {IGlobalArgs} from "@chainsafe/lodestar-cli/src/options"; +import {KEY_IMPORTED_PREFIX} from "@chainsafe/lodestar-keymanager-server"; + +export async function getAndInitValidatorsWithKeystore({ + node, + keystoreContent, + keystorePubKey, + useRestApi, + testLoggerOpts, +}: { + node: BeaconNode; + keystoreContent: string; + keystorePubKey: string; + useRestApi?: boolean; + testLoggerOpts?: TestLoggerOpts; +}): Promise<{ + validator: Validator; + secretKeys: SecretKey[]; + keystoreContent: string; + signers: SignerLocal[]; + slashingProtection: ISlashingProtection; + tempDirs: { + keystoreDir: DirResult; + passwordFile: FileResult; + }; +}> { + const keystoreDir = tmp.dirSync({unsafeCleanup: true}); + const keystoreFile = path.join(`${keystoreDir.name}`, `${KEY_IMPORTED_PREFIX}_${keystorePubKey}.json`); + + fs.writeFileSync(keystoreFile, keystoreContent, {encoding: "utf8", flag: "wx"}); + + const passwordFile = tmp.fileSync(); + fs.writeFileSync(passwordFile.name, "test123!", {encoding: "utf8"}); + + const vcConfig = { + network: "prater", + importKeystoresPath: [`${keystoreDir.name}`], + importKeystoresPassword: `${passwordFile.name}`, + keymanagerEnabled: true, + keymanagerAuthEnabled: true, + keymanagerHost: "127.0.0.1", + keymanagerPort: 9666, + keymanagerCors: "*", + }; + + const logger = testLogger("Vali", testLoggerOpts); + const tmpDir = tmp.dirSync({unsafeCleanup: true}); + const dbOps = { + config: node.config, + controller: new LevelDbController({name: tmpDir.name}, {logger}), + }; + const slashingProtection = new SlashingProtection(dbOps); + + const signers: SignerLocal[] = []; + + const {secretKeys, unlockSecretKeys: _unlockSecretKeys} = await getLocalSecretKeys( + (vcConfig as unknown) as IValidatorCliArgs & IGlobalArgs + ); + if (secretKeys.length > 0) { + // Log pubkeys for auditing + logger.info(`Decrypted ${secretKeys.length} local keystores`); + for (const secretKey of secretKeys) { + logger.info(secretKey.toPublicKey().toHex()); + signers.push({ + type: SignerType.Local, + secretKey, + }); + } + } + + const validator = await Validator.initializeFromBeaconNode({ + dbOps, + api: useRestApi ? getNodeApiUrl(node) : node.api, + slashingProtection, + logger, + signers, + }); + + return { + validator, + secretKeys, + keystoreContent, + signers, + slashingProtection, + tempDirs: { + keystoreDir: keystoreDir, + passwordFile: passwordFile, + }, + }; +} export async function getAndInitDevValidators({ node, @@ -75,6 +176,6 @@ export async function getAndInitDevValidators({ function getNodeApiUrl(node: BeaconNode): string { const host = node.opts.api.rest.host || "127.0.0.1"; - const port = node.opts.api.rest.port || 9596; + const port = node.opts.api.rest.port || 19596; return `http://${host}:${port}`; } diff --git a/packages/validator/src/index.ts b/packages/validator/src/index.ts index 2b1a56a9df9..6f9a1b9b7a3 100644 --- a/packages/validator/src/index.ts +++ b/packages/validator/src/index.ts @@ -3,8 +3,8 @@ */ export {Validator, ValidatorOptions} from "./validator"; +export {ValidatorStore, SignerType, Signer, SignerLocal, SignerRemote} from "./services/validatorStore"; export {waitForGenesis} from "./genesis"; -export {SignerType, Signer, SignerLocal, SignerRemote} from "./services/validatorStore"; // Remote signer client export {externalSignerGetKeys, externalSignerPostSignature, externalSignerUpCheck} from "./util/externalSignerClient"; diff --git a/packages/validator/src/services/attestation.ts b/packages/validator/src/services/attestation.ts index 85b9ed3f2d1..c950afc0d21 100644 --- a/packages/validator/src/services/attestation.ts +++ b/packages/validator/src/services/attestation.ts @@ -11,6 +11,7 @@ import {IndicesService} from "./indices"; import {toHexString} from "@chainsafe/ssz"; import {ChainHeaderTracker, HeadEventData} from "./chainHeaderTracker"; import {ValidatorEvent, ValidatorEventEmitter} from "./emitter"; +import {PubkeyHex} from "../types"; /** * Service that sets up and handles validator attester duties. @@ -40,6 +41,10 @@ export class AttestationService { clock.runEverySlot(this.runAttestationTasks); } + removeDutiesForKey(pubkey: PubkeyHex): void { + this.dutiesService.removeDutiesForKey(pubkey); + } + private runAttestationTasks = async (slot: Slot, signal: AbortSignal): Promise => { // Fetch info first so a potential delay is absorved by the sleep() below const dutiesByCommitteeIndex = groupAttDutiesByCommitteeIndex(this.dutiesService.getDutiesAtSlot(slot)); diff --git a/packages/validator/src/services/attestationDuties.ts b/packages/validator/src/services/attestationDuties.ts index 8f6c8fddd69..8182de1a084 100644 --- a/packages/validator/src/services/attestationDuties.ts +++ b/packages/validator/src/services/attestationDuties.ts @@ -8,6 +8,7 @@ import {IndicesService} from "./indices"; import {IClock, extendError, ILoggerVc} from "../util"; import {ValidatorStore} from "./validatorStore"; import {ChainHeaderTracker, HeadEventData} from "./chainHeaderTracker"; +import {PubkeyHex} from "../types"; /** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. */ const HISTORICAL_DUTIES_EPOCHS = 2; @@ -46,6 +47,19 @@ export class AttestationDutiesService { chainHeadTracker.runOnNewHead(this.onNewHead); } + removeDutiesForKey(pubkey: PubkeyHex): void { + for (const [epoch, attDutiesAtEpoch] of this.dutiesByIndexByEpoch) { + for (const [vIndex, attDutyAndProof] of attDutiesAtEpoch.dutiesByIndex) { + if (toHexString(attDutyAndProof.duty.pubkey) === pubkey) { + attDutiesAtEpoch.dutiesByIndex.delete(vIndex); + if (attDutiesAtEpoch.dutiesByIndex.size === 0) { + this.dutiesByIndexByEpoch.delete(epoch); + } + } + } + } + } + /** Returns all `ValidatorDuty` for the given `slot` */ getDutiesAtSlot(slot: Slot): AttDutyAndProof[] { const epoch = computeEpochAtSlot(slot); diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index db48f8cf8c0..ca8b778ac37 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -7,6 +7,7 @@ import {Api} from "@chainsafe/lodestar-api"; import {IClock, extendError, ILoggerVc} from "../util"; import {ValidatorStore} from "./validatorStore"; import {BlockDutiesService, GENESIS_SLOT} from "./blockDuties"; +import {PubkeyHex} from "../types"; /** * Service that sets up and handles validator block proposal duties. @@ -25,6 +26,10 @@ export class BlockProposingService { this.dutiesService = new BlockDutiesService(logger, api, clock, validatorStore, this.notifyBlockProductionFn); } + removeDutiesForKey(pubkey: PubkeyHex): void { + this.dutiesService.removeDutiesForKey(pubkey); + } + /** * `BlockDutiesService` must call this fn to trigger block creation * This function may run more than once at a time, rationale in `BlockDutiesService.pollBeaconProposers` diff --git a/packages/validator/src/services/blockDuties.ts b/packages/validator/src/services/blockDuties.ts index c0b7034bc21..037f211af2d 100644 --- a/packages/validator/src/services/blockDuties.ts +++ b/packages/validator/src/services/blockDuties.ts @@ -4,6 +4,7 @@ import {toHexString} from "@chainsafe/ssz"; import {Api, routes} from "@chainsafe/lodestar-api"; import {IClock, extendError, differenceHex, ILoggerVc} from "../util"; import {ValidatorStore} from "./validatorStore"; +import {PubkeyHex} from "../types"; /** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch */ const HISTORICAL_DUTIES_EPOCHS = 2; @@ -58,6 +59,14 @@ export class BlockDutiesService { return Array.from(publicKeys.values()); } + removeDutiesForKey(pubkey: PubkeyHex): void { + for (const blockDutyAtEpoch of this.proposers.values()) { + blockDutyAtEpoch.data = blockDutyAtEpoch.data.filter((proposer) => { + return toHexString(proposer.pubkey) !== pubkey; + }); + } + } + private runBlockDutiesTask = async (slot: Slot): Promise => { try { if (slot < 0) { diff --git a/packages/validator/src/services/indices.ts b/packages/validator/src/services/indices.ts index 6ec3681c80c..376a05d6374 100644 --- a/packages/validator/src/services/indices.ts +++ b/packages/validator/src/services/indices.ts @@ -53,6 +53,16 @@ export class IndicesService { return this.pollValidatorIndicesPromise; } + removeDutiesForKey(pubkey: PubkeyHex): void { + for (const [key, value] of this.index2pubkey) { + if (value === pubkey) { + this.index2pubkey.delete(key); + } + } + + this.pubkey2index.delete(pubkey); + } + /** Iterate through all the voting pubkeys in the `ValidatorStore` and attempt to learn any unknown validator indices. Returns the new discovered indexes */ private async pollValidatorIndicesInternal(): Promise { diff --git a/packages/validator/src/services/syncCommittee.ts b/packages/validator/src/services/syncCommittee.ts index 39ebe85089a..52eac60e63a 100644 --- a/packages/validator/src/services/syncCommittee.ts +++ b/packages/validator/src/services/syncCommittee.ts @@ -10,6 +10,7 @@ import {SyncCommitteeDutiesService, SyncDutyAndProofs} from "./syncCommitteeDuti import {groupSyncDutiesBySubcommitteeIndex, SubcommitteeDuty} from "./utils"; import {IndicesService} from "./indices"; import {ChainHeaderTracker} from "./chainHeaderTracker"; +import {PubkeyHex} from "../types"; /** * Service that sets up and handles validator sync duties. @@ -32,6 +33,10 @@ export class SyncCommitteeService { clock.runEverySlot(this.runSyncCommitteeTasks); } + removeDutiesForKey(pubkey: PubkeyHex): void { + this.dutiesService.removeDutiesForKey(pubkey); + } + private runSyncCommitteeTasks = async (slot: Slot, signal: AbortSignal): Promise => { try { // Before altair fork no need to check duties diff --git a/packages/validator/src/services/syncCommitteeDuties.ts b/packages/validator/src/services/syncCommitteeDuties.ts index aada6f57f4c..3d3123af45e 100644 --- a/packages/validator/src/services/syncCommitteeDuties.ts +++ b/packages/validator/src/services/syncCommitteeDuties.ts @@ -11,6 +11,7 @@ import {Api, routes} from "@chainsafe/lodestar-api"; import {IndicesService} from "./indices"; import {IClock, extendError, ILoggerVc} from "../util"; import {ValidatorStore} from "./validatorStore"; +import {PubkeyHex} from "../types"; /** Only retain `HISTORICAL_DUTIES_PERIODS` duties prior to the current periods. */ const HISTORICAL_DUTIES_PERIODS = 2; @@ -91,6 +92,19 @@ export class SyncCommitteeDutiesService { return duties; } + removeDutiesForKey(pubkey: PubkeyHex): void { + for (const [syncPeriod, validatorDutyAtPeriodMap] of this.dutiesByIndexByPeriod) { + for (const [validatorIndex, dutyAtPeriod] of validatorDutyAtPeriodMap) { + if (toHexString(dutyAtPeriod.duty.pubkey) === pubkey) { + validatorDutyAtPeriodMap.delete(validatorIndex); + if (validatorDutyAtPeriodMap.size === 0) { + this.dutiesByIndexByPeriod.delete(syncPeriod); + } + } + } + } + } + private runDutiesTasks = async (currentEpoch: Epoch): Promise => { // Before altair fork (+ lookahead) no need to check duties if (currentEpoch < this.config.ALTAIR_FORK_EPOCH - ALTAIR_FORK_LOOKAHEAD_EPOCHS) { @@ -230,7 +244,7 @@ export class SyncCommitteeDutiesService { private async getSelectionProofs(slot: Slot, duty: routes.validator.SyncDuty): Promise { // Fast indexing with precomputed pubkeyHex. Fallback to toHexString(duty.pubkey) - const pubkey = this.indicesService.index2pubkey.get(duty.validatorIndex) ?? duty.pubkey; + const pubkey = this.indicesService.index2pubkey.get(duty.validatorIndex) ?? toHexString(duty.pubkey); const dutiesAndProofs: SyncSelectionProof[] = []; for (const index of duty.validatorSyncCommitteeIndices) { diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index cd194035077..8d2fbfcb833 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -30,7 +30,7 @@ import { } from "@chainsafe/lodestar-types"; import {fromHexString, List, toHexString} from "@chainsafe/ssz"; import {routes} from "@chainsafe/lodestar-api"; -import {ISlashingProtection} from "../slashingProtection"; +import {Interchange, InterchangeFormatVersion, ISlashingProtection} from "../slashingProtection"; import {PubkeyHex} from "../types"; import {getAggregationBits} from "./utils"; import {externalSignerPostSignature} from "../util/externalSignerClient"; @@ -72,13 +72,25 @@ export class ValidatorStore { genesis: phase0.Genesis ) { for (const signer of signers) { - this.validators.set(getSignerPubkeyHex(signer), signer); + this.addSigner(signer); } this.slashingProtection = slashingProtection; this.genesisValidatorsRoot = genesis.genesisValidatorsRoot; } + addSigner(signer: Signer): void { + this.validators.set(getSignerPubkeyHex(signer), signer); + } + + getSigner(pubkeyHex: PubkeyHex): Signer | undefined { + return this.validators.get(pubkeyHex); + } + + removeSigner(pubkeyHex: PubkeyHex): boolean { + return this.validators.delete(pubkeyHex); + } + /** Return true if there is at least 1 pubkey registered */ hasSomeValidators(): boolean { return this.validators.size > 0; @@ -92,6 +104,14 @@ export class ValidatorStore { return this.validators.has(pubkeyHex); } + async importInterchange(interchange: Interchange): Promise { + return this.slashingProtection.importInterchange(interchange, this.genesisValidatorsRoot); + } + + async exportInterchange(pubkeys: BLSPubkey[], formatVersion: InterchangeFormatVersion): Promise { + return this.slashingProtection.exportInterchange(this.genesisValidatorsRoot, pubkeys, formatVersion); + } + async signBlock( pubkey: BLSPubkey, block: allForks.BeaconBlock, diff --git a/packages/validator/src/slashingProtection/interface.ts b/packages/validator/src/slashingProtection/interface.ts index ce2413429d5..9b4d27043e3 100644 --- a/packages/validator/src/slashingProtection/interface.ts +++ b/packages/validator/src/slashingProtection/interface.ts @@ -1,4 +1,5 @@ -import {BLSPubkey} from "@chainsafe/lodestar-types"; +import {BLSPubkey, Root} from "@chainsafe/lodestar-types"; +import {Interchange, InterchangeFormatVersion} from "./interchange/types"; import {SlashingProtectionBlock, SlashingProtectionAttestation} from "./types"; export interface ISlashingProtection { @@ -10,4 +11,11 @@ export interface ISlashingProtection { * Check an attestation for slash safety, and if it is safe, record it in the database */ checkAndInsertAttestation(pubKey: BLSPubkey, attestation: SlashingProtectionAttestation): Promise; + + importInterchange(interchange: Interchange, genesisValidatorsRoot: Uint8Array | Root): Promise; + exportInterchange( + genesisValidatorsRoot: Uint8Array | Root, + pubkeys: BLSPubkey[], + formatVersion: InterchangeFormatVersion + ): Promise; } diff --git a/packages/validator/src/types.ts b/packages/validator/src/types.ts index c45095a7341..56d090379d2 100644 --- a/packages/validator/src/types.ts +++ b/packages/validator/src/types.ts @@ -14,7 +14,14 @@ export type BLSKeypair = { secretKey: SecretKey; }; +/** + * The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._ + * ``` + * "0x93247f2209abcacf57b75a51dafae777f9dd38bc7053d1af526f220a7489a6d3a2753e5f3e8b1cfe39b56f43611df74a" + * ``` + */ export type PubkeyHex = string; + export type LodestarValidatorDatabaseController = Pick< IDatabaseController, "get" | "start" | "values" | "batchPut" | "keys" | "get" | "put" diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index e9a0406b77d..c7201ce9f00 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -19,6 +19,7 @@ import {toHexString} from "@chainsafe/ssz"; import {ValidatorEventEmitter} from "./services/emitter"; import {ValidatorStore, Signer} from "./services/validatorStore"; import {computeEpochAtSlot, getCurrentSlot} from "@chainsafe/lodestar-beacon-state-transition"; +import {PubkeyHex} from "./types"; export type ValidatorOptions = { slashingProtection: ISlashingProtection; @@ -44,16 +45,20 @@ type State = {status: Status.running; controller: AbortController} | {status: St * Main class for the Validator client. */ export class Validator { + readonly validatorStore: ValidatorStore; + private readonly blockProposingService: BlockProposingService; + private readonly attestationService: AttestationService; + private readonly syncCommitteeService: SyncCommitteeService; + private readonly indicesService: IndicesService; private readonly config: IBeaconConfig; private readonly api: Api; private readonly clock: IClock; private readonly emitter: ValidatorEventEmitter; private readonly chainHeaderTracker: ChainHeaderTracker; - private readonly validatorStore: ValidatorStore; private readonly logger: ILogger; private state: State = {status: Status.stopped}; - constructor(opts: ValidatorOptions, genesis: Genesis) { + constructor(opts: ValidatorOptions, readonly genesis: Genesis) { const {dbOps, logger, slashingProtection, signers, graffiti} = opts; const config = createIBeaconConfig(dbOps.config, genesis.genesisValidatorsRoot); @@ -69,13 +74,32 @@ export class Validator { const clock = new Clock(config, logger, {genesisTime: Number(genesis.genesisTime)}); const validatorStore = new ValidatorStore(config, slashingProtection, signers, genesis); - const indicesService = new IndicesService(logger, api, validatorStore); + this.indicesService = new IndicesService(logger, api, validatorStore); this.emitter = new ValidatorEventEmitter(); this.chainHeaderTracker = new ChainHeaderTracker(logger, api, this.emitter); const loggerVc = getLoggerVc(logger, clock); - new BlockProposingService(config, loggerVc, api, clock, validatorStore, graffiti); - new AttestationService(loggerVc, api, clock, validatorStore, this.emitter, indicesService, this.chainHeaderTracker); - new SyncCommitteeService(config, loggerVc, api, clock, validatorStore, this.chainHeaderTracker, indicesService); + + this.blockProposingService = new BlockProposingService(config, loggerVc, api, clock, validatorStore, graffiti); + + this.attestationService = new AttestationService( + loggerVc, + api, + clock, + validatorStore, + this.emitter, + this.indicesService, + this.chainHeaderTracker + ); + + this.syncCommitteeService = new SyncCommitteeService( + config, + loggerVc, + api, + clock, + validatorStore, + this.chainHeaderTracker, + this.indicesService + ); this.config = config; this.logger = logger; @@ -107,6 +131,13 @@ export class Validator { return new Validator(opts, genesis); } + removeDutiesForKey(pubkey: PubkeyHex): void { + this.indicesService.removeDutiesForKey(pubkey); + this.blockProposingService.removeDutiesForKey(pubkey); + this.attestationService.removeDutiesForKey(pubkey); + this.syncCommitteeService.removeDutiesForKey(pubkey); + } + /** * Instantiates block and attestation services and runs them once the chain has been started. */ diff --git a/packages/validator/test/unit/services/attestationDuties.test.ts b/packages/validator/test/unit/services/attestationDuties.test.ts index 0b21377b402..521692da658 100644 --- a/packages/validator/test/unit/services/attestationDuties.test.ts +++ b/packages/validator/test/unit/services/attestationDuties.test.ts @@ -126,4 +126,65 @@ describe("AttestationDutiesService", function () { "prepareBeaconCommitteeSubnet() must be called once after getting the duties" ); }); + + it("Should remove signer from attestation duties", async function () { + // Reply with an active validator that has an index + const validatorResponse = { + ...defaultValidator, + index, + validator: {...defaultValidator.validator, pubkey: pubkeys[0]}, + }; + api.beacon.getStateValidators.resolves({data: [validatorResponse]}); + + // Reply with some duties + const slot = 1; + const duty: routes.validator.AttesterDuty = { + slot: slot, + committeeIndex: 1, + committeeLength: 120, + committeesAtSlot: 120, + validatorCommitteeIndex: 1, + validatorIndex: index, + pubkey: pubkeys[0], + }; + api.validator.getAttesterDuties.resolves({dependentRoot: ZERO_HASH, data: [duty]}); + + // Accept all subscriptions + api.validator.prepareBeaconCommitteeSubnet.resolves(); + + // Clock will call runAttesterDutiesTasks() immediatelly + const clock = new ClockMock(); + const indicesService = new IndicesService(logger, api, validatorStore); + const dutiesService = new AttestationDutiesService( + loggerVc, + api, + clock, + validatorStore, + indicesService, + chainHeadTracker + ); + + // Trigger clock onSlot for slot 0 + await clock.tickEpochFns(0, controller.signal); + + // first confirm duties for this and next epoch should be persisted + expect(Object.fromEntries(dutiesService["dutiesByIndexByEpoch"].get(0)?.dutiesByIndex || new Map())).to.deep.equal( + { + 4: {duty: duty, selectionProof: null}, + }, + "Wrong dutiesService.attesters Map at current epoch" + ); + expect(Object.fromEntries(dutiesService["dutiesByIndexByEpoch"].get(1)?.dutiesByIndex || new Map())).to.deep.equal( + { + 4: {duty: duty, selectionProof: null}, + }, + "Wrong dutiesService.attesters Map at current epoch" + ); + // then remove + dutiesService.removeDutiesForKey(toHexString(pubkeys[0])); + expect(Object.fromEntries(dutiesService["dutiesByIndexByEpoch"])).to.deep.equal( + {}, + "Wrong dutiesService.attesters Map at current epoch after removal" + ); + }); }); diff --git a/packages/validator/test/unit/services/blockDuties.test.ts b/packages/validator/test/unit/services/blockDuties.test.ts index 6aa2cd7298c..5f8e44a05f2 100644 --- a/packages/validator/test/unit/services/blockDuties.test.ts +++ b/packages/validator/test/unit/services/blockDuties.test.ts @@ -23,7 +23,7 @@ describe("BlockDutiesService", function () { let pubkeys: Uint8Array[]; // Initialize pubkeys in before() so bls is already initialized before(() => { - const secretKeys = Array.from({length: 2}, (_, i) => bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1))); + const secretKeys = Array.from({length: 3}, (_, i) => bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1))); pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); validatorStore.votingPubkeys.returns(pubkeys.map(toHexString)); validatorStore.hasVotingPubkey.returns(true); @@ -111,4 +111,50 @@ describe("BlockDutiesService", function () { "Second call to notifyBlockProductionFn() after the re-org with pubkey[1]" ); }); + + it("Should remove signer from duty", async function () { + // Reply with some duties + const slot = 0; // genesisTime is right now, so test with slot = currentSlot + const duties: ProposerDutiesRes = { + dependentRoot: ZERO_HASH, + data: [ + {slot: slot, validatorIndex: 0, pubkey: pubkeys[0]}, + {slot: slot, validatorIndex: 1, pubkey: pubkeys[1]}, + {slot: 33, validatorIndex: 2, pubkey: pubkeys[2]}, + ], + }; + + const dutiesRemoved: ProposerDutiesRes = { + dependentRoot: ZERO_HASH, + data: [ + {slot: slot, validatorIndex: 1, pubkey: pubkeys[1]}, + {slot: 33, validatorIndex: 2, pubkey: pubkeys[2]}, + ], + }; + api.validator.getProposerDuties.resolves(duties); + + const notifyBlockProductionFn = sinon.stub(); // Returns void + + const clock = new ClockMock(); + const dutiesService = new BlockDutiesService(loggerVc, api, clock, validatorStore, notifyBlockProductionFn); + + // Trigger clock onSlot for slot 0 + await clock.tickSlotFns(0, controller.signal); + await clock.tickSlotFns(32, controller.signal); + + // first confirm the duties for the epochs was persisted + expect(Object.fromEntries(dutiesService["proposers"])).to.deep.equal( + {0: duties, 1: duties}, + "Wrong dutiesService.proposers Map" + ); + + // then remove a signers public key + dutiesService.removeDutiesForKey(toHexString(pubkeys[0])); + + // confirm that the duties no longer contain the signers public key + expect(Object.fromEntries(dutiesService["proposers"])).to.deep.equal( + {0: dutiesRemoved, 1: dutiesRemoved}, + "Wrong dutiesService.proposers Map" + ); + }); }); diff --git a/packages/validator/test/unit/services/indicesService.test.ts b/packages/validator/test/unit/services/indicesService.test.ts new file mode 100644 index 00000000000..889206dee52 --- /dev/null +++ b/packages/validator/test/unit/services/indicesService.test.ts @@ -0,0 +1,59 @@ +import {toBufferBE} from "bigint-buffer"; +import {expect} from "chai"; +import sinon from "sinon"; +import bls from "@chainsafe/bls"; +import {toHexString} from "@chainsafe/ssz"; +import {ValidatorStore} from "../../../src/services/validatorStore"; +import {getApiClientStub} from "../../utils/apiStub"; +import {testLogger} from "../../utils/logger"; +import {IndicesService} from "../../../src/services/indices"; + +describe("IndicesService", function () { + const sandbox = sinon.createSandbox(); + const logger = testLogger(); + const api = getApiClientStub(sandbox); + const validatorStore = sinon.createStubInstance(ValidatorStore) as ValidatorStore & + sinon.SinonStubbedInstance; + + let pubkeys: Uint8Array[]; // Initialize pubkeys in before() so bls is already initialized + + before(() => { + const secretKeys = [ + bls.SecretKey.fromBytes(toBufferBE(BigInt(98), 32)), + bls.SecretKey.fromBytes(toBufferBE(BigInt(99), 32)), + ]; + pubkeys = secretKeys.map((sk) => sk.toPublicKey().toBytes()); + }); + + it("Should remove pubkey", async function () { + const indicesService = new IndicesService(logger, api, validatorStore); + const firstValidatorIndex = 0; + const secondValidatorIndex = 1; + + const pubkey1 = toHexString(pubkeys[firstValidatorIndex]); + const pubkey2 = toHexString(pubkeys[secondValidatorIndex]); + + indicesService.index2pubkey.set(firstValidatorIndex, pubkey1); + indicesService.index2pubkey.set(secondValidatorIndex, pubkey2); + + indicesService.pubkey2index.set(pubkey1, firstValidatorIndex); + indicesService.pubkey2index.set(pubkey2, secondValidatorIndex); + + // remove pubkey2 + indicesService.removeDutiesForKey(pubkey2); + + expect(Object.fromEntries(indicesService.index2pubkey)).to.deep.equal( + { + "0": `${pubkey1}`, + }, + "Wrong indicesService.index2pubkey Map" + ); + + expect(Object.fromEntries(indicesService.pubkey2index)).to.deep.equal( + { + [`${pubkey1}`]: 0, + }, + "Wrong indicesService.pubkey2index Map" + ); + }); +}); diff --git a/packages/validator/test/unit/services/syncCommitteDuties.test.ts b/packages/validator/test/unit/services/syncCommitteDuties.test.ts index dd3601a9115..0cdd39bb0d8 100644 --- a/packages/validator/test/unit/services/syncCommitteDuties.test.ts +++ b/packages/validator/test/unit/services/syncCommitteDuties.test.ts @@ -186,4 +186,70 @@ describe("SyncCommitteeDutiesService", function () { "Wrong dutiesService.dutiesByIndexByPeriod Map" ); }); + + it("Should remove signer from sync committee duties", async function () { + // Reply with some duties + const duty1: routes.validator.SyncDuty = { + pubkey: pubkeys[0], + validatorIndex: indices[0], + validatorSyncCommitteeIndices: [7], + }; + const duty2: routes.validator.SyncDuty = { + pubkey: pubkeys[1], + validatorIndex: indices[1], + validatorSyncCommitteeIndices: [7], + }; + api.validator.getSyncCommitteeDuties + .withArgs(sinon.match.any, sinon.match.any) + .resolves({dependentRoot: ZERO_HASH, data: [duty1, duty2]}); + + // Accept all subscriptions + api.validator.prepareSyncCommitteeSubnets.resolves(); + + // Clock will call runAttesterDutiesTasks() immediatelly + const clock = new ClockMock(); + const indicesService = new IndicesService(logger, api, validatorStore); + const dutiesService = new SyncCommitteeDutiesService(config, loggerVc, api, clock, validatorStore, indicesService); + + // Trigger clock onSlot for slot 0 + await clock.tickEpochFns(0, controller.signal); + + // Duties for this and next epoch should be persisted + const dutiesByIndexByPeriodObj = Object.fromEntries( + Array.from(dutiesService["dutiesByIndexByPeriod"].entries()).map(([period, dutiesByIndex]) => [ + period, + Object.fromEntries(dutiesByIndex), + ]) + ); + expect(dutiesByIndexByPeriodObj).to.deep.equal( + { + 0: { + [indices[0]]: {dependentRoot: ZERO_HASH, duty: duty1}, + [indices[1]]: {dependentRoot: ZERO_HASH, duty: duty2}, + }, + 1: { + [indices[0]]: {dependentRoot: ZERO_HASH, duty: duty1}, + [indices[1]]: {dependentRoot: ZERO_HASH, duty: duty2}, + }, + }, + "Wrong dutiesService.dutiesByIndexByPeriod Map" + ); + // then remove signer with pubkeys[0] + dutiesService.removeDutiesForKey(toHexString(pubkeys[0])); + + // Removed public key should be removed from duties for this and next epoch should be persisted + const dutiesByIndexByPeriodObjAfterRemoval = Object.fromEntries( + Array.from(dutiesService["dutiesByIndexByPeriod"].entries()).map(([period, dutiesByIndex]) => [ + period, + Object.fromEntries(dutiesByIndex), + ]) + ); + expect(dutiesByIndexByPeriodObjAfterRemoval).to.deep.equal( + { + 0: {[indices[1]]: {dependentRoot: ZERO_HASH, duty: duty2}}, + 1: {[indices[1]]: {dependentRoot: ZERO_HASH, duty: duty2}}, + }, + "Wrong dutiesService.dutiesByIndexByPeriod Map" + ); + }); }); diff --git a/yarn.lock b/yarn.lock index 9a39fe66181..c4c863caed7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -413,7 +413,7 @@ buffer "^5.4.3" randombytes "^2.1.0" -"@chainsafe/bls-keystore@2.0.0": +"@chainsafe/bls-keystore@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@chainsafe/bls-keystore/-/bls-keystore-2.0.0.tgz#6ac15109579c2caeae62c83c0257add0e8be2508" integrity sha512-XGtgGKdjYqKP09SUsfwaStsYuWuXB56/614dC1XhggG4LH8KTrFOjxb9SkS+T1BUu5doCXd9YA+gNLy01zv+Ww== @@ -5222,6 +5222,13 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" +fastify-bearer-auth@6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/fastify-bearer-auth/-/fastify-bearer-auth-6.1.0.tgz#906780002e32b556fbc3dce3e14861896812fd32" + integrity sha512-qplgYoQ1OipyAMwSlAQVxEhlRnBtYukF6sLmxmfBqFgCdHPkvpZ0gL/AsQfpKFt31wlq4fuHZIymMNaWHVfgcA== + dependencies: + fastify-plugin "^3.0.0" + fastify-cors@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/fastify-cors/-/fastify-cors-6.0.1.tgz#300e0f1cbeedda19d9de284e9cf05c65842c182b"