Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: worker friendly logger instances #5490

Merged
merged 19 commits into from
May 15, 2023
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/beacon-node/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@
"@lodestar/db": "^1.8.0",
"@lodestar/fork-choice": "^1.8.0",
"@lodestar/light-client": "^1.8.0",
"@lodestar/logger": "^1.8.0",
"@lodestar/params": "^1.8.0",
"@lodestar/reqresp": "^1.8.0",
"@lodestar/state-transition": "^1.8.0",
Expand Down
7 changes: 4 additions & 3 deletions packages/beacon-node/src/network/discv5/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@ import {exportToProtobuf} from "@libp2p/peer-id-factory";
import {createKeypairFromPeerId, ENR, ENRData, IKeypair, SignableENR} from "@chainsafe/discv5";
import {spawn, Thread, Worker} from "@chainsafe/threads";
import {chainConfigFromJson, chainConfigToJson, BeaconConfig} from "@lodestar/config";
import {Logger} from "@lodestar/utils";
import {LoggerNode} from "@lodestar/logger/node";
import {NetworkCoreMetrics} from "../core/metrics.js";
import {Discv5WorkerApi, Discv5WorkerData, LodestarDiscv5Opts} from "./types.js";

export type Discv5Opts = {
peerId: PeerId;
discv5: LodestarDiscv5Opts;
logger: Logger;
logger: LoggerNode;
config: BeaconConfig;
metrics?: NetworkCoreMetrics;
};
Expand All @@ -29,7 +29,7 @@ type Discv5WorkerStatus =
* Wrapper class abstracting the details of discv5 worker instantiation and message-passing
*/
export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<EventEmitter, Discv5Events>}) {
private logger: Logger;
private logger: LoggerNode;
private status: Discv5WorkerStatus;
private keypair: IKeypair;

Expand All @@ -53,6 +53,7 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<E
metrics: Boolean(this.opts.metrics),
chainConfig: chainConfigFromJson(chainConfigToJson(this.opts.config)),
genesisValidatorsRoot: this.opts.config.genesisValidatorsRoot,
loggerOpts: this.opts.logger.toOpts(),
};
const worker = new Worker("./worker.js", {workerData} as ConstructorParameters<typeof Worker>[1]);

Expand Down
2 changes: 2 additions & 0 deletions packages/beacon-node/src/network/discv5/types.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import {Discv5, ENRData, SignableENRData} from "@chainsafe/discv5";
import {Observable} from "@chainsafe/threads/observable";
import {ChainConfig} from "@lodestar/config";
import {LoggerNodeOpts} from "@lodestar/logger/node";

// TODO export IDiscv5Config so we don't need this convoluted type
type Discv5Config = Parameters<(typeof Discv5)["create"]>[0]["config"];
Expand All @@ -22,6 +23,7 @@ export interface Discv5WorkerData {
metrics: boolean;
chainConfig: ChainConfig;
genesisValidatorsRoot: Uint8Array;
loggerOpts: LoggerNodeOpts;
}

/**
Expand Down
9 changes: 9 additions & 0 deletions packages/beacon-node/src/network/discv5/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {expose} from "@chainsafe/threads/worker";
import {Observable, Subject} from "@chainsafe/threads/observable";
import {createKeypairFromPeerId, Discv5, ENR, ENRData, SignableENR, SignableENRData} from "@chainsafe/discv5";
import {createBeaconConfig, BeaconConfig} from "@lodestar/config";
import {getNodeLogger} from "@lodestar/logger/node";
import {RegistryMetricCreator} from "../../metrics/index.js";
import {collectNodeJSMetrics} from "../../metrics/nodeJsMetrics.js";
import {ENRKey} from "../metadata.js";
Expand Down Expand Up @@ -55,6 +56,8 @@ const workerData = worker.workerData as Discv5WorkerData;
// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
if (!workerData) throw Error("workerData must be defined");

const logger = getNodeLogger(workerData.loggerOpts);

// Set up metrics, nodejs and discv5-specific
let metricsRegistry: RegistryMetricCreator | undefined;
let enrRelevanceMetric: Gauge<"status"> | undefined;
Expand Down Expand Up @@ -134,3 +137,9 @@ const module: Discv5WorkerApi = {
};

expose(module);

logger.info("discv5 worker started", {
peerId: peerId.toString(),
listenAddr: workerData.bindAddr,
initialENR: workerData.enr,
});
9 changes: 5 additions & 4 deletions packages/beacon-node/src/network/network.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ import {Connection} from "@libp2p/interface-connection";
import {PeerId} from "@libp2p/interface-peer-id";
import {Multiaddr} from "@multiformats/multiaddr";
import {BeaconConfig} from "@lodestar/config";
import {Logger, sleep, toHex} from "@lodestar/utils";
import {sleep, toHex} from "@lodestar/utils";
import {ForkName} from "@lodestar/params";
import {LoggerNode} from "@lodestar/logger/node";
import {computeEpochAtSlot, computeTimeAtSlot} from "@lodestar/state-transition";
import {Epoch, phase0, allForks} from "@lodestar/types";
import {routes} from "@lodestar/api";
Expand Down Expand Up @@ -42,7 +43,7 @@ type NetworkModules = {
opts: NetworkOptions;
config: BeaconConfig;
libp2p: Libp2p;
logger: Logger;
logger: LoggerNode;
chain: IBeaconChain;
signal: AbortSignal;
peersData: PeersData;
Expand All @@ -63,7 +64,7 @@ export type NetworkInitModules = {
config: BeaconConfig;
peerId: PeerId;
peerStoreDir?: string;
logger: Logger;
logger: LoggerNode;
metrics: Metrics | null;
chain: IBeaconChain;
reqRespHandlers: ReqRespHandlers;
Expand All @@ -88,7 +89,7 @@ export class Network implements INetwork {
private readonly statusCache: LocalStatusCache;
private readonly libp2p: Libp2p;
private readonly gossipsub: Eth2Gossipsub;
private readonly logger: Logger;
private readonly logger: LoggerNode;
private readonly config: BeaconConfig;
private readonly clock: IClock;
private readonly chain: IBeaconChain;
Expand Down
7 changes: 4 additions & 3 deletions packages/beacon-node/src/network/peers/discover.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ import {PeerId} from "@libp2p/interface-peer-id";
import {Multiaddr} from "@multiformats/multiaddr";
import {PeerInfo} from "@libp2p/interface-peer-info";
import {BeaconConfig} from "@lodestar/config";
import {Logger, pruneSetToMax, sleep} from "@lodestar/utils";
import {pruneSetToMax, sleep} from "@lodestar/utils";
import {ENR} from "@chainsafe/discv5";
import {ATTESTATION_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";
import {LoggerNode} from "@lodestar/logger/node";
import {NetworkCoreMetrics} from "../core/metrics.js";
import {Libp2p} from "../interface.js";
import {ENRKey, SubnetType} from "../metadata.js";
Expand All @@ -30,7 +31,7 @@ export type PeerDiscoveryModules = {
libp2p: Libp2p;
peerRpcScores: IPeerRpcScoreStore;
metrics: NetworkCoreMetrics | null;
logger: Logger;
logger: LoggerNode;
config: BeaconConfig;
};

Expand Down Expand Up @@ -76,7 +77,7 @@ export class PeerDiscovery {
private libp2p: Libp2p;
private peerRpcScores: IPeerRpcScoreStore;
private metrics: NetworkCoreMetrics | null;
private logger: Logger;
private logger: LoggerNode;
private config: BeaconConfig;
private cachedENRs = new Map<PeerIdStr, CachedENR>();
private randomNodeQuery: QueryStatus = {code: QueryStatusCode.NotActive};
Expand Down
6 changes: 3 additions & 3 deletions packages/beacon-node/src/network/peers/peerManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {BitArray} from "@chainsafe/ssz";
import {SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params";
import {BeaconConfig} from "@lodestar/config";
import {allForks, altair, phase0} from "@lodestar/types";
import {Logger} from "@lodestar/utils";
import {LoggerNode} from "@lodestar/logger/node";
import {GoodByeReasonCode, GOODBYE_KNOWN_CODES, Libp2pEvent} from "../../constants/index.js";
import {NetworkCoreMetrics} from "../core/metrics.js";
import {NetworkEvent, INetworkEventBus} from "../events.js";
Expand Down Expand Up @@ -83,7 +83,7 @@ export type PeerManagerOpts = {

export type PeerManagerModules = {
libp2p: Libp2p;
logger: Logger;
logger: LoggerNode;
metrics: NetworkCoreMetrics | null;
reqResp: IReqRespBeaconNode;
gossip: Eth2Gossipsub;
Expand Down Expand Up @@ -115,7 +115,7 @@ enum RelevantPeerStatus {
*/
export class PeerManager {
private libp2p: Libp2p;
private logger: Logger;
private logger: LoggerNode;
private metrics: NetworkCoreMetrics | null;
private reqResp: IReqRespBeaconNode;
private gossipsub: Eth2Gossipsub;
Expand Down
4 changes: 2 additions & 2 deletions packages/beacon-node/src/node/nodejs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {Registry} from "prom-client";
import {PeerId} from "@libp2p/interface-peer-id";
import {BeaconConfig} from "@lodestar/config";
import {phase0} from "@lodestar/types";
import {Logger} from "@lodestar/utils";
import {LoggerNode} from "@lodestar/logger/node";
import {Api, ServerApi} from "@lodestar/api";
import {BeaconStateAllForks} from "@lodestar/state-transition";
import {ProcessShutdownCallback} from "@lodestar/validator";
Expand Down Expand Up @@ -45,7 +45,7 @@ export type BeaconNodeInitModules = {
opts: IBeaconNodeOptions;
config: BeaconConfig;
db: IBeaconDb;
logger: Logger;
logger: LoggerNode;
processShutdownCallback: ProcessShutdownCallback;
peerId: PeerId;
peerStoreDir?: string;
Expand Down
3 changes: 2 additions & 1 deletion packages/beacon-node/test/sim/merge-interop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ import fs from "node:fs";
import {Context} from "mocha";
import {fromHexString} from "@chainsafe/ssz";
import {isExecutionStateType, isMergeTransitionComplete} from "@lodestar/state-transition";
import {LogLevel, sleep, TimestampFormatCode} from "@lodestar/utils";
import {LogLevel, sleep} from "@lodestar/utils";
import {TimestampFormatCode} from "@lodestar/logger";
import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params";
import {ChainConfig} from "@lodestar/config";
import {routes} from "@lodestar/api";
Expand Down
33 changes: 4 additions & 29 deletions packages/beacon-node/test/utils/logger.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
import winston from "winston";
import {createWinstonLogger, Logger, LogLevel, TimestampFormat} from "@lodestar/utils";
import {LogLevel} from "@lodestar/utils";
import {getEnvLogger, LoggerEnvOpts} from "@lodestar/logger";
export {LogLevel};

export type TestLoggerOpts = {
logLevel?: LogLevel;
logFile?: string;
timestampFormat?: TimestampFormat;
};
export type TestLoggerOpts = LoggerEnvOpts;

/**
* Run the test with ENVs to control log level:
Expand All @@ -16,25 +12,4 @@ export type TestLoggerOpts = {
* VERBOSE=1 mocha .ts
* ```
*/
export function testLogger(module?: string, opts?: TestLoggerOpts): Logger {
const transports: winston.transport[] = [
new winston.transports.Console({level: getLogLevelFromEnvs() || opts?.logLevel || LogLevel.error}),
];
if (opts?.logFile) {
transports.push(
new winston.transports.File({
level: LogLevel.debug,
filename: opts.logFile,
})
);
}

return createWinstonLogger({module, ...opts}, transports);
}

function getLogLevelFromEnvs(): LogLevel | null {
if (process.env["LOG_LEVEL"]) return process.env["LOG_LEVEL"] as LogLevel;
if (process.env["DEBUG"]) return LogLevel.debug;
if (process.env["VERBOSE"]) return LogLevel.verbose;
return null;
}
export const testLogger = getEnvLogger;
1 change: 1 addition & 0 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
"@lodestar/config": "^1.8.0",
"@lodestar/db": "^1.8.0",
"@lodestar/light-client": "^1.8.0",
"@lodestar/logger": "^1.8.0",
"@lodestar/params": "^1.8.0",
"@lodestar/state-transition": "^1.8.0",
"@lodestar/types": "^1.8.0",
Expand Down
14 changes: 8 additions & 6 deletions packages/cli/src/cmds/beacon/handler.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import path from "node:path";
import {Registry} from "prom-client";
import {ErrorAborted, Logger} from "@lodestar/utils";
import {ErrorAborted} from "@lodestar/utils";
import {LevelDbController} from "@lodestar/db";
import {BeaconNode, BeaconDb} from "@lodestar/beacon-node";
import {ChainForkConfig, createBeaconConfig} from "@lodestar/config";
import {ACTIVE_PRESET, PresetName} from "@lodestar/params";
import {ProcessShutdownCallback} from "@lodestar/validator";
import {LoggerNode, getNodeLogger} from "@lodestar/logger";

import {GlobalArgs, parseBeaconNodeArgs} from "../../options/index.js";
import {BeaconNodeOptions, getBeaconConfigFromArgs} from "../../config/index.js";
import {getNetworkBootnodes, getNetworkData, isKnownNetworkName, readBootnodes} from "../../networks/index.js";
import {onGracefulShutdown, getCliLogger, mkdir, writeFile600Perm, cleanOldLogFiles} from "../../util/index.js";
import {onGracefulShutdown, mkdir, writeFile600Perm, cleanOldLogFiles, parseLoggerArgs} from "../../util/index.js";
import {getVersionData} from "../../util/version.js";
import {BeaconArgs} from "./options.js";
import {getBeaconPaths} from "./paths.js";
Expand Down Expand Up @@ -149,12 +150,13 @@ export async function beaconHandlerInit(args: BeaconArgs & GlobalArgs) {
return {config, options, beaconPaths, network, version, commit, peerId, logger};
}

export function initLogger(args: BeaconArgs, dataDir: string, config: ChainForkConfig): Logger {
const {logger, logParams} = getCliLogger(args, {defaultLogFilepath: path.join(dataDir, "beacon.log")}, config);
export function initLogger(args: BeaconArgs, dataDir: string, config: ChainForkConfig): LoggerNode {
const defaultLogFilepath = path.join(dataDir, "beacon.log");
const logger = getNodeLogger(parseLoggerArgs(args, {defaultLogFilepath}, config));
try {
cleanOldLogFiles(logParams.filename, logParams.rotateMaxFiles);
cleanOldLogFiles(args, {defaultLogFilepath});
} catch (e) {
logger.debug("Not able to delete log files", logParams, e as Error);
logger.debug("Not able to delete log files", {}, e as Error);
}

return logger;
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/cmds/beacon/options.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {Options} from "yargs";
import {beaconNodeOptions, paramsOptions, BeaconNodeArgs} from "../../options/index.js";
import {logOptions} from "../../options/logOptions.js";
import {CliCommandOptions, LogArgs} from "../../util/index.js";
import {LogArgs, logOptions} from "../../options/logOptions.js";
import {CliCommandOptions} from "../../util/index.js";
import {defaultBeaconPaths, BeaconPaths} from "./paths.js";

type BeaconExtraArgs = {
Expand Down
7 changes: 5 additions & 2 deletions packages/cli/src/cmds/lightclient/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,20 @@ import {ApiError, getClient} from "@lodestar/api";
import {Lightclient} from "@lodestar/light-client";
import {fromHexString} from "@chainsafe/ssz";
import {LightClientRestTransport} from "@lodestar/light-client/transport";
import {getNodeLogger} from "@lodestar/logger";
import {getBeaconConfigFromArgs} from "../../config/beaconParams.js";
import {getGlobalPaths} from "../../paths/global.js";
import {parseLoggerArgs} from "../../util/logger.js";
import {GlobalArgs} from "../../options/index.js";
import {getCliLogger} from "../../util/index.js";
import {ILightClientArgs} from "./options.js";

export async function lightclientHandler(args: ILightClientArgs & GlobalArgs): Promise<void> {
const {config, network} = getBeaconConfigFromArgs(args);
const globalPaths = getGlobalPaths(args, network);

const {logger} = getCliLogger(args, {defaultLogFilepath: path.join(globalPaths.dataDir, "lightclient.log")}, config);
const logger = getNodeLogger(
parseLoggerArgs(args, {defaultLogFilepath: path.join(globalPaths.dataDir, "lightclient.log")}, config)
);
const {beaconApiUrl, checkpointRoot} = args;
const api = getClient({baseUrl: beaconApiUrl}, {config});
const res = await api.beacon.getGenesis();
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/cmds/lightclient/options.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import {logOptions} from "../../options/logOptions.js";
import {CliCommandOptions, LogArgs} from "../../util/index.js";
import {LogArgs, logOptions} from "../../options/logOptions.js";
import {CliCommandOptions} from "../../util/index.js";

export type ILightClientArgs = LogArgs & {
beaconApiUrl: string;
Expand Down
14 changes: 6 additions & 8 deletions packages/cli/src/cmds/validator/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ import {
} from "@lodestar/validator";
import {getMetrics, MetricsRegister} from "@lodestar/validator";
import {RegistryMetricCreator, collectNodeJSMetrics, HttpMetricsServer, MonitoringService} from "@lodestar/beacon-node";
import {getNodeLogger} from "@lodestar/logger";
import {getBeaconConfigFromArgs} from "../../config/index.js";
import {GlobalArgs} from "../../options/index.js";
import {YargsError, getDefaultGraffiti, mkdir, getCliLogger, cleanOldLogFiles} from "../../util/index.js";
import {YargsError, cleanOldLogFiles, getDefaultGraffiti, mkdir, parseLoggerArgs} from "../../util/index.js";
import {onGracefulShutdown, parseFeeRecipient, parseProposerConfig} from "../../util/index.js";
import {getVersionData} from "../../util/version.js";
import {getAccountPaths, getValidatorPaths} from "./paths.js";
Expand All @@ -35,15 +36,12 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr
const validatorPaths = getValidatorPaths(args, network);
const accountPaths = getAccountPaths(args, network);

const {logger, logParams} = getCliLogger(
args,
{defaultLogFilepath: path.join(validatorPaths.dataDir, "validator.log")},
config
);
const defaultLogFilepath = path.join(validatorPaths.dataDir, "validator.log");
const logger = getNodeLogger(parseLoggerArgs(args, {defaultLogFilepath}, config));
try {
cleanOldLogFiles(logParams.filename, logParams.rotateMaxFiles);
cleanOldLogFiles(args, {defaultLogFilepath});
} catch (e) {
logger.debug("Not able to delete log files", logParams, e as Error);
logger.debug("Not able to delete log files", {}, e as Error);
}

const persistedKeysBackend = new PersistedKeysBackend(accountPaths);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Logger} from "@lodestar/utils";
import {LogLevel, Logger} from "@lodestar/utils";
import {LocalKeystoreDefinition} from "../interface.js";

export type DecryptKeystoreWorkerAPI = {
Expand All @@ -10,5 +10,5 @@ export type KeystoreDecryptOptions = {
onDecrypt?: (index: number) => void;
// Try to use the cache file if it exists
cacheFilePath?: string;
logger: Pick<Logger, "info" | "warn" | "debug">;
logger: Pick<Logger, LogLevel.info | LogLevel.warn | LogLevel.debug>;
};
4 changes: 2 additions & 2 deletions packages/cli/src/cmds/validator/options.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {defaultOptions} from "@lodestar/validator";
import {logOptions} from "../../options/logOptions.js";
import {ensure0xPrefix, CliCommandOptions, LogArgs} from "../../util/index.js";
import {LogArgs, logOptions} from "../../options/logOptions.js";
import {ensure0xPrefix, CliCommandOptions} from "../../util/index.js";
import {keymanagerRestApiServerOptsDefault} from "./keymanager/server.js";
import {defaultAccountPaths, defaultValidatorPaths} from "./paths.js";

Expand Down
Loading