From 65b0e5fefe53d46c9c2114a16ca2462c48ec6352 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:28:28 +0200 Subject: [PATCH 1/3] Make dataDir optional --- packages/cli/src/options/globalOptions.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/options/globalOptions.ts b/packages/cli/src/options/globalOptions.ts index 0262399e675..a732a042a35 100644 --- a/packages/cli/src/options/globalOptions.ts +++ b/packages/cli/src/options/globalOptions.ts @@ -1,9 +1,10 @@ +import {ACTIVE_PRESET} from "@lodestar/params"; import {NetworkName, networkNames} from "../networks/index.js"; import {ICliCommandOptions, readFile} from "../util/index.js"; import {paramsOptions, IParamsArgs} from "./paramsOptions.js"; interface IGlobalSingleArgs { - dataDir: string; + dataDir?: string; network?: NetworkName; paramsFile: string; preset: string; @@ -33,6 +34,7 @@ const globalSingleOptions: ICliCommandOptions = { preset: { hidden: true, type: "string", + default: ACTIVE_PRESET, }, }; From fe4b40f4ab3e48b2575f560dc6a5df344b55ecfc Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:33:18 +0200 Subject: [PATCH 2/3] Update getCliLogger consumers --- packages/cli/src/cmds/beacon/handler.ts | 2 +- packages/cli/src/cmds/lightclient/handler.ts | 7 +++++-- packages/cli/src/cmds/validator/handler.ts | 3 ++- .../cli/src/cmds/validator/slashingProtection/export.ts | 5 +++-- .../cli/src/cmds/validator/slashingProtection/import.ts | 4 +++- packages/cli/src/util/logger.ts | 6 +++--- packages/cli/test/unit/util/loggerTransport.test.ts | 2 +- 7 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/cmds/beacon/handler.ts b/packages/cli/src/cmds/beacon/handler.ts index d78040a082d..e76b344fe00 100644 --- a/packages/cli/src/cmds/beacon/handler.ts +++ b/packages/cli/src/cmds/beacon/handler.ts @@ -30,7 +30,7 @@ export async function beaconHandler(args: IBeaconArgs & IGlobalArgs): Promise { abortController.abort(); diff --git a/packages/cli/src/cmds/lightclient/handler.ts b/packages/cli/src/cmds/lightclient/handler.ts index cdf66956b0a..72995f25581 100644 --- a/packages/cli/src/cmds/lightclient/handler.ts +++ b/packages/cli/src/cmds/lightclient/handler.ts @@ -1,15 +1,18 @@ +import path from "node:path"; import {getClient} from "@lodestar/api"; import {Lightclient} from "@lodestar/light-client"; import {fromHexString} from "@chainsafe/ssz"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; +import {getGlobalPaths} from "../../paths/global.js"; import {IGlobalArgs} from "../../options/index.js"; import {getCliLogger} from "../../util/index.js"; import {ILightClientArgs} from "./options.js"; export async function lightclientHandler(args: ILightClientArgs & IGlobalArgs): Promise { - const {config} = getBeaconConfigFromArgs(args); + const {config, network} = getBeaconConfigFromArgs(args); + const globalPaths = getGlobalPaths(args, network); - const logger = getCliLogger(args, {defaultLogFile: "lightclient.log"}, config); + const logger = getCliLogger(args, {defaultLogFilepath: path.join(globalPaths.dataDir, "lightclient.log")}, config); const {beaconApiUrl, checkpointRoot} = args; const api = getClient({baseUrl: beaconApiUrl}, {config}); const {data: genesisData} = await api.beacon.getGenesis(); diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index ee4308e820f..a83c7fe0bcd 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -1,3 +1,4 @@ +import path from "node:path"; import {setMaxListeners} from "node:events"; import {LevelDbController} from "@lodestar/db"; import {ProcessShutdownCallback, SlashingProtection, Validator, ValidatorProposerConfig} from "@lodestar/validator"; @@ -28,7 +29,7 @@ export async function validatorHandler(args: IValidatorCliArgs & IGlobalArgs): P const validatorPaths = getValidatorPaths(args, network); const accountPaths = getAccountPaths(args, network); - const logger = getCliLogger(args, {defaultLogFile: "validator.log"}, config); + const logger = getCliLogger(args, {defaultLogFilepath: path.join(validatorPaths.dataDir, "validator.log")}, config); const persistedKeysBackend = new PersistedKeysBackend(accountPaths); const valProposerConfig = getProposerConfigFromArgs(args, {persistedKeysBackend, accountPaths}); diff --git a/packages/cli/src/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/validator/slashingProtection/export.ts index ab0d2ec6d2e..4f212c33fb7 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/export.ts @@ -1,3 +1,4 @@ +import path from "node:path"; import {InterchangeFormatVersion} from "@lodestar/validator"; import {ICliCommand, writeFile600Perm} from "../../../util/index.js"; import {IGlobalArgs} from "../../../options/index.js"; @@ -39,9 +40,9 @@ export const exportCmd: ICliCommand< handler: async (args) => { const {config, network} = getBeaconConfigFromArgs(args); - + const validatorPaths = getValidatorPaths(args, network); // slashingProtection commands are fast so do not require logFile feature - const logger = getCliLogger(args, {defaultLogFile: "validator.log"}, config); + const logger = getCliLogger(args, {defaultLogFilepath: path.join(validatorPaths.dataDir, "validator.log")}, config); const {validatorsDbDir: dbPath} = getValidatorPaths(args, network); diff --git a/packages/cli/src/cmds/validator/slashingProtection/import.ts b/packages/cli/src/cmds/validator/slashingProtection/import.ts index 875de060cc4..c3ceff66203 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/import.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/import.ts @@ -1,4 +1,5 @@ import fs from "node:fs"; +import path from "node:path"; import {Interchange} from "@lodestar/validator"; import {ICliCommand} from "../../../util/index.js"; import {IGlobalArgs} from "../../../options/index.js"; @@ -40,8 +41,9 @@ export const importCmd: ICliCommand< handler: async (args) => { const {config, network} = getBeaconConfigFromArgs(args); + const validatorPaths = getValidatorPaths(args, network); // slashingProtection commands are fast so do not require logFile feature - const logger = getCliLogger(args, {defaultLogFile: "validator.log"}, config); + const logger = getCliLogger(args, {defaultLogFilepath: path.join(validatorPaths.dataDir, "validator.log")}, config); const {validatorsDbDir: dbPath} = getValidatorPaths(args, network); diff --git a/packages/cli/src/util/logger.ts b/packages/cli/src/util/logger.ts index a617f2bd051..c00aa30dddf 100644 --- a/packages/cli/src/util/logger.ts +++ b/packages/cli/src/util/logger.ts @@ -37,12 +37,12 @@ export interface ILogArgs { */ export function getCliLogger( args: ILogArgs & Pick, - paths: {defaultLogFile: string}, + paths: {defaultLogFilepath: string}, config: IChainForkConfig, opts?: {hideTimestamp?: boolean} ): ILogger { const consoleTransport = new ConsoleDynamicLevel({ - // Set defaultLevel, not level for dynamic level setting of ConsoleDynamicLevel + // Set defaultLevel, not level for dynamic level setting of ConsoleDynamicLvevel defaultLevel: args.logLevel ?? LOG_LEVEL_DEFAULT, debugStdout: true, handleExceptions: true, @@ -72,7 +72,7 @@ export function getCliLogger( // `lodestar --logFileDailyRotate 10` -> set daily rotate to custom value 10 // `lodestar --logFileDailyRotate 0` -> disable daily rotate and accumulate in same file const rotateMaxFiles = args.logFileDailyRotate ?? LOG_DAILY_ROTATE_DEFAULT; - const filename = args.logFile ?? path.join(args.dataDir, paths.defaultLogFile); + const filename = args.logFile ?? paths.defaultLogFilepath; const logFileLevel = args.logFileLevel ?? LOG_FILE_LEVEL_DEFAULT; transports.push( diff --git a/packages/cli/test/unit/util/loggerTransport.test.ts b/packages/cli/test/unit/util/loggerTransport.test.ts index 6c0f9b84d46..d4a02c51b48 100644 --- a/packages/cli/test/unit/util/loggerTransport.test.ts +++ b/packages/cli/test/unit/util/loggerTransport.test.ts @@ -139,7 +139,7 @@ describe("winston transport log to file", () => { }); function getCliLoggerTest(logArgs: Partial): ReturnType { - return getCliLogger({dataDir: "", ...logArgs}, {defaultLogFile: ""}, config, {hideTimestamp: true}); + return getCliLogger(logArgs, {defaultLogFilepath: ""}, config, {hideTimestamp: true}); } /** Wait for file to exist have some content, then return its contents */ From 4290751abdd890301a78060b300cdb0788ce25e9 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:39:19 +0200 Subject: [PATCH 3/3] Run dev command to test --- .../test/e2e/importKeystoresFromApi.test.ts | 2 +- packages/cli/test/e2e/runDevCmd.test.ts | 32 +++++++++++++ packages/cli/test/e2e/voluntaryExit.test.ts | 2 +- .../test/unit/util/loggerTransport.test.ts | 9 +++- packages/cli/test/utils/childprocRunner.ts | 45 +++++++++++++------ .../cli/test/utils/keymanagerTestRunners.ts | 2 +- 6 files changed, 73 insertions(+), 19 deletions(-) create mode 100644 packages/cli/test/e2e/runDevCmd.test.ts diff --git a/packages/cli/test/e2e/importKeystoresFromApi.test.ts b/packages/cli/test/e2e/importKeystoresFromApi.test.ts index b9e6a1638ae..a3464a18b63 100644 --- a/packages/cli/test/e2e/importKeystoresFromApi.test.ts +++ b/packages/cli/test/e2e/importKeystoresFromApi.test.ts @@ -66,7 +66,7 @@ describeCliTest("import keystores from api", function ({spawnCli}) { ); // Attempt to run a second process and expect the keystore lock to throw - const vcProc2 = spawnCli([ + const vcProc2 = spawnCli({pipeStdToParent: true, logPrefix: "vc-2"}, [ // ⏎ "validator", `--dataDir=${dataDir}`, diff --git a/packages/cli/test/e2e/runDevCmd.test.ts b/packages/cli/test/e2e/runDevCmd.test.ts new file mode 100644 index 00000000000..b9e2187b0da --- /dev/null +++ b/packages/cli/test/e2e/runDevCmd.test.ts @@ -0,0 +1,32 @@ +import {getClient} from "@lodestar/api"; +import {config} from "@lodestar/config/default"; +import {retry} from "@lodestar/utils"; +import {describeCliTest} from "../utils/childprocRunner.js"; +import {itDone} from "../utils/runUtils.js"; + +describeCliTest("Run dev command", function ({spawnCli}) { + itDone("Run dev command with no --dataDir until beacon api is listening", async function (done) { + const beaconPort = 39011; + + const devProc = spawnCli({pipeStdToParent: false, printOnlyOnError: true, logPrefix: "dev"}, [ + // ⏎ + "dev", + "--reset", + "--startValidators=0..7", + `--rest.port=${beaconPort}`, + ]); + + // Exit early if process exits + devProc.on("exit", (code) => { + if (code !== null && code > 0) { + done(Error(`process exited with code ${code}`)); + } + }); + + const beaconUrl = `http://localhost:${beaconPort}`; + const client = getClient({baseUrl: beaconUrl}, {config}); + + // Wrap in retry since the API may not be listening yet + await retry(() => client.node.getHealth(), {retryDelay: 1000, retries: 60}); + }); +}); diff --git a/packages/cli/test/e2e/voluntaryExit.test.ts b/packages/cli/test/e2e/voluntaryExit.test.ts index f555732dd32..35cc699629f 100644 --- a/packages/cli/test/e2e/voluntaryExit.test.ts +++ b/packages/cli/test/e2e/voluntaryExit.test.ts @@ -13,7 +13,7 @@ describeCliTest("voluntaryExit cmd", function ({spawnCli}) { itDone("Perform a voluntary exit", async function (done) { const restPort = 9596; - const devBnProc = spawnCli([ + const devBnProc = spawnCli({pipeStdToParent: false, logPrefix: "dev"}, [ // ⏎ "dev", `--dataDir=${path.join(testFilesDir, "dev-voluntary-exit")}`, diff --git a/packages/cli/test/unit/util/loggerTransport.test.ts b/packages/cli/test/unit/util/loggerTransport.test.ts index d4a02c51b48..b6537b6ca96 100644 --- a/packages/cli/test/unit/util/loggerTransport.test.ts +++ b/packages/cli/test/unit/util/loggerTransport.test.ts @@ -4,7 +4,7 @@ import rimraf from "rimraf"; import {expect} from "chai"; import {config} from "@lodestar/config/default"; import {LodestarError, LogData, LogFormat, logFormats, LogLevel} from "@lodestar/utils"; -import {getCliLogger, ILogArgs} from "../../../src/util/logger.js"; +import {getCliLogger, ILogArgs, LOG_FILE_DISABLE_KEYWORD} from "../../../src/util/logger.js"; describe("winston logger format and options", () => { interface ITestCase { @@ -139,7 +139,12 @@ describe("winston transport log to file", () => { }); function getCliLoggerTest(logArgs: Partial): ReturnType { - return getCliLogger(logArgs, {defaultLogFilepath: ""}, config, {hideTimestamp: true}); + return getCliLogger( + {logFile: LOG_FILE_DISABLE_KEYWORD, ...logArgs}, + {defaultLogFilepath: "logger_transport_test.log"}, + config, + {hideTimestamp: true} + ); } /** Wait for file to exist have some content, then return its contents */ diff --git a/packages/cli/test/utils/childprocRunner.ts b/packages/cli/test/utils/childprocRunner.ts index 18d793ea380..5e3a79e5e7a 100644 --- a/packages/cli/test/utils/childprocRunner.ts +++ b/packages/cli/test/utils/childprocRunner.ts @@ -11,11 +11,14 @@ const cliLibScriptPath = esmRelativePathJoin("../../lib/index.js"); /* eslint-disable no-console */ export type DescribeArgs = { - spawnCli(args: string[]): child_process.ChildProcessWithoutNullStreams; + spawnCli(opts: SpawnCliOpts, args: string[]): child_process.ChildProcessWithoutNullStreams; }; type SpawnCliOpts = { ensureProcRunning?: boolean; + logPrefix?: string; + pipeStdToParent?: boolean; + printOnlyOnError?: boolean; }; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type @@ -35,8 +38,8 @@ export function describeCliTest(testName: string, callback: (this: Mocha.Suite, }); const args: DescribeArgs = { - spawnCli(args: string[], opts?: SpawnCliOpts) { - const proc = spawnCli(args); + spawnCli(opts: SpawnCliOpts, args: string[]) { + const proc = spawnCli(opts, args); console.log(`Created process ${proc.pid}`); afterEachCallbacks.push(async function () { @@ -70,9 +73,9 @@ export function describeCliTest(testName: string, callback: (this: Mocha.Suite, }); } -export function spawnCli(lodestarArgs: string[]): child_process.ChildProcessWithoutNullStreams { - // TODO: Customize - const logPrefix = "vc"; +export function spawnCli(opts: SpawnCliOpts, lodestarArgs: string[]): child_process.ChildProcessWithoutNullStreams { + let stdstr = ""; + const logPrefix = opts?.logPrefix ?? ""; const command = RUN_FROM_SRC ? // ts-node --esm cli.ts @@ -87,17 +90,31 @@ export function spawnCli(lodestarArgs: string[]): child_process.ChildProcessWith const proc = child_process.spawn(command, prefixArgs); - proc.stdout.on("data", (chunk) => { - const str = Buffer.from(chunk).toString("utf8"); - process.stdout.write(`${logPrefix} ${proc.pid}: ${str}`); // str already contains a new line. console.log adds a new line - }); - proc.stderr.on("data", (chunk) => { - const str = Buffer.from(chunk).toString("utf8"); - process.stderr.write(`${logPrefix} ${proc.pid}: ${str}`); // str already contains a new line. console.log adds a new line - }); + if (opts?.pipeStdToParent) { + proc.stdout.on("data", (chunk) => { + const str = Buffer.from(chunk).toString("utf8"); + process.stdout.write(`${logPrefix} ${proc.pid}: ${str}`); // str already contains a new line. console.log adds a new line + }); + proc.stderr.on("data", (chunk) => { + const str = Buffer.from(chunk).toString("utf8"); + process.stderr.write(`${logPrefix} ${proc.pid}: ${str}`); // str already contains a new line. console.log adds a new line + }); + } else { + proc.stdout.on("data", (chunk) => { + stdstr += Buffer.from(chunk).toString("utf8"); + }); + proc.stderr.on("data", (chunk) => { + stdstr += Buffer.from(chunk).toString("utf8"); + }); + } proc.on("exit", (code) => { console.log("process exited", {code}); + if (!opts?.pipeStdToParent) { + if (!opts?.printOnlyOnError || (code !== null && code > 0)) { + console.log(stdstr); + } + } }); return proc; diff --git a/packages/cli/test/utils/keymanagerTestRunners.ts b/packages/cli/test/utils/keymanagerTestRunners.ts index c2dff534438..e1cc0d3f5d6 100644 --- a/packages/cli/test/utils/keymanagerTestRunners.ts +++ b/packages/cli/test/utils/keymanagerTestRunners.ts @@ -37,7 +37,7 @@ export function getKeymanagerTestRunner({args: {spawnCli}, afterEachCallbacks, d afterEachCallbacks.push(() => beaconServer.close()); await beaconServer.listen(); - const validatorProc = spawnCli([ + const validatorProc = spawnCli({pipeStdToParent: true, logPrefix: "vc"}, [ // ⏎ "validator", `--dataDir=${dataDir}`,