diff --git a/plugins/plugin-codeflare-dashboard/src/components/Dashboard/Grid.tsx b/plugins/plugin-codeflare-dashboard/src/components/Dashboard/Grid.tsx index 3de95878..ca361001 100644 --- a/plugins/plugin-codeflare-dashboard/src/components/Dashboard/Grid.tsx +++ b/plugins/plugin-codeflare-dashboard/src/components/Dashboard/Grid.tsx @@ -17,8 +17,8 @@ import React from "react" import { Box, BoxProps, Spacer, Text, TextProps } from "ink" -import type { Props, State } from "./index.js" -import type { UpdatePayload, Worker } from "./types.js" +import type { Props } from "./index.js" +import type { Worker } from "./types.js" import { avg } from "./stats.js" @@ -30,7 +30,7 @@ type GridProps = { scale: Props["scale"] title: NonNullable["title"] states: NonNullable["states"] - workers: State["workers"][number] + workers: Worker[] } export default class Grid extends React.PureComponent { @@ -74,7 +74,7 @@ export default class Grid extends React.PureComponent { } /** @return current `Worker[]` model */ - private get workers(): UpdatePayload["workers"] { + private get workers(): Worker[] { return this.props.workers || [] } diff --git a/plugins/plugin-codeflare-dashboard/src/components/Dashboard/index.tsx b/plugins/plugin-codeflare-dashboard/src/components/Dashboard/index.tsx index 7b774397..6ac25ba6 100644 --- a/plugins/plugin-codeflare-dashboard/src/components/Dashboard/index.tsx +++ b/plugins/plugin-codeflare-dashboard/src/components/Dashboard/index.tsx @@ -18,10 +18,11 @@ import React from "react" import prettyMillis from "pretty-ms" import { Box, Spacer, Text } from "ink" -import type { GridSpec, UpdatePayload, Worker } from "./types.js" +import type { GridSpec, UpdatePayload, LogLineUpdate, WorkersUpdate, Worker } from "./types.js" import Grid from "./Grid.js" import Timeline from "./Timeline.js" +import { isWorkersUpdate } from "./types.js" export type Props = { /** CodeFlare Profile for this dashboard */ @@ -33,42 +34,39 @@ export type Props = { /** Scale up the grid? [default: 1] */ scale?: number + /** Grid models, where null means to insert a line break */ grids: (null | GridSpec)[] } -export type State = { - /** millis since epoch of the first update */ - firstUpdate: number +export type State = Pick & + LogLineUpdate & { + /** millis since epoch of the first update */ + firstUpdate: number - /** millis since epoch of the last update */ - lastUpdate: number + /** millis since epoch of the last update */ + lastUpdate: number - /** iteration count to help us keep "last updated ago" UI fresh */ - iter: number + /** iteration count to help us keep "last updated ago" UI fresh */ + iter: number - /** interval to keep "last updated ago" UI fresh */ - agoInterval: ReturnType + /** interval to keep "last updated ago" UI fresh */ + agoInterval: ReturnType - /** Lines of raw output to be displayed */ - events: UpdatePayload["events"] + /** Controller that allows us to shut down gracefully */ + watchers: { quit: () => void }[] - /** Controller that allows us to shut down gracefully */ - watchers: { quit: () => void }[] - - /** - * Model of current workers; outer idx is grid index; inner idx is - * worker idx, i.e. for each grid, we have an array of Workers. - */ - workers: Worker[][] -} + /** + * Model of current workers; outer idx is grid index; inner idx is + * worker idx, i.e. for each grid, we have an array of Workers. + */ + workers: Worker[][] + } export default class Dashboard extends React.PureComponent { public componentDidMount() { this.setState({ workers: [], - watchers: this.gridModels.map((props, gridIdx) => - props.initWatcher((model: UpdatePayload) => this.onUpdate(gridIdx, model)) - ), + watchers: this.gridModels.map((props, gridIdx) => props.initWatcher((model) => this.onUpdate(gridIdx, model))), agoInterval: setInterval(() => this.setState((curState) => ({ iter: (curState?.iter || 0) + 1 })), 5 * 1000), }) } @@ -87,8 +85,15 @@ export default class Dashboard extends React.PureComponent { this.setState((curState) => ({ firstUpdate: (curState && curState.firstUpdate) || Date.now(), // TODO pull from the events lastUpdate: Date.now(), // TODO pull from the events - events: !model.events || model.events.length === 0 ? curState?.events : model.events, - workers: !curState?.workers + events: !isWorkersUpdate(model) + ? curState?.events + : !model.events || model.events.length === 0 + ? curState?.events + : model.events, + logLine: !isWorkersUpdate(model) ? model.logLine : curState?.logLine, + workers: !isWorkersUpdate(model) + ? curState?.workers + : !curState?.workers ? [model.workers] : [...curState.workers.slice(0, gridIdx), model.workers, ...curState.workers.slice(gridIdx + 1)], })) @@ -102,10 +107,15 @@ export default class Dashboard extends React.PureComponent { } /** @return current `events` model */ - private get events(): UpdatePayload["events"] { + private get events(): State["events"] { return this.state?.events } + /** @return current `logLine` model */ + private get logLine(): State["logLine"] { + return this.state?.logLine + } + /** @return first update time */ private get firstUpdate() { return this.state?.firstUpdate || Date.now() @@ -184,17 +194,20 @@ export default class Dashboard extends React.PureComponent { /** Render log lines and events */ private footer() { - if (!this.events) { + if (!this.events && !this.logLine) { return } else { - const rows = this.events.map(({ line, timestamp }) => { - // the controller (controller/dashboard/utilization/Live) - // leaves a {timestamp} breadcrumb in the raw line text, so - // that we,as the view, can inject a "5m ago" text, while - // preserving the ansi formatting that surrounds the timestamp - const txt = line.replace("{timestamp}", () => this.agos(timestamp)) - return {txt} - }) + const rows = (this.events || []) + .map(({ line, timestamp }) => { + // the controller (controller/dashboard/utilization/Live) + // leaves a {timestamp} breadcrumb in the raw line text, so + // that we,as the view, can inject a "5m ago" text, while + // preserving the ansi formatting that surrounds the timestamp + const txt = line.replace("{timestamp}", () => this.agos(timestamp)) + return {txt} + }) + .concat((this.logLine ? [this.logLine] : []).map((line) => {line})) + return ( {rows} diff --git a/plugins/plugin-codeflare-dashboard/src/components/Dashboard/types.ts b/plugins/plugin-codeflare-dashboard/src/components/Dashboard/types.ts index a3256ec0..478a8c12 100644 --- a/plugins/plugin-codeflare-dashboard/src/components/Dashboard/types.ts +++ b/plugins/plugin-codeflare-dashboard/src/components/Dashboard/types.ts @@ -37,8 +37,12 @@ export type Worker = { lastUpdate: number } -/** Model that allows the controllers to pass updated `Worker` info */ -export type UpdatePayload = { +export type LogLineUpdate = { + /** Log lines */ + logLine: string +} + +export type WorkersUpdate = { /** Per-worker status info */ workers: Worker[] @@ -46,6 +50,13 @@ export type UpdatePayload = { events?: { line: string; timestamp: number }[] } +/** Model that allows the controllers to pass updated `Worker` info */ +export type UpdatePayload = LogLineUpdate | WorkersUpdate + +export function isWorkersUpdate(update: UpdatePayload): update is WorkersUpdate { + return Array.isArray((update as WorkersUpdate).workers) +} + /** Callback from controller when it has updated data */ export type OnData = (payload: UpdatePayload) => void diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/kinds.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/kinds.ts index 5a8c97bd..720f79ac 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/kinds.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/kinds.ts @@ -16,23 +16,38 @@ import type { SupportedGrid } from "./grids.js" -type Kind = SupportedGrid | "logs" +type Kind = SupportedGrid | "logs" | "env" export type KindA = Kind | "all" export default Kind -export const resourcePaths: Record = { +/** A filepath with a `Kind` discriminant to help understand the content of the `filepath` */ +export type KindedSource = { kind: Kind; filepath: string } + +/** + * A source to be tailf'd is either a string (the filepath to the + * source) or that plus a `Kind` discriminant. + */ +type Source = string | KindedSource + +/** Extract the `filepath` property of `source` */ +export function filepathOf(source: Source) { + return typeof source === "string" ? source : source.filepath +} + +export const resourcePaths: Record = { status: [ "events/kubernetes.txt", "events/job-status.txt", "events/pods.txt", "events/runtime-env-setup.txt", - // "logs/job.txt", + { kind: "logs", filepath: "logs/job.txt" }, ], "gpu%": ["resources/gpu.txt"], "gpumem%": ["resources/gpu.txt"], "cpu%": ["resources/pod-vmstat.txt"], "mem%": ["resources/pod-memory.txt"], logs: ["logs/job.txt"], + env: ["env.json"], } export function validKinds() { diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/options.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/options.ts index d8ec65f1..2cfcf16b 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/options.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/options.ts @@ -26,11 +26,14 @@ type Options = { /** Number of lines of events to show [default: 8] */ events: number + + /** Number of lines of application logs to show [default: 1] */ + lines: number } export default Options export const flags = { boolean: ["demo"], - alias: { events: ["e"], theme: ["t"], demo: ["d"], scale: ["s"] }, + alias: { events: ["e"], lines: ["l"], theme: ["t"], demo: ["d"], scale: ["s"] }, } diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/status/Live.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/status/Live.ts index cfeed944..5cc02dc5 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/status/Live.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/status/Live.ts @@ -40,6 +40,9 @@ export default class Live { /** Number of lines of event output to retain. TODO this depends on height of terminal? */ private static readonly MAX_HEAP = 1000 + /** Model of logLines. TODO circular buffer and obey options.lines */ + // private logLine = "" + /** Model of the lines of output */ private readonly events = new Heap((a, b) => { if (a.line === b.line) { @@ -62,9 +65,13 @@ export default class Live { private readonly opts: Pick ) { tails.map((tailf) => { - tailf.then(({ stream }) => { + tailf.then(({ kind, stream }) => { stream.on("data", (data) => { if (data) { + if (kind === "logs") { + this.pushLineAndPublish(data, cb) + } + const line = stripAnsi(data) const cols = line.split(/\s+/) @@ -77,7 +84,6 @@ export default class Live { if (!name || !timestamp) { // console.error("Bad status record", line) - // this.pushEventAndPublish(data, metric, timestamp, cb) return } else if (!metric) { // ignoring this line @@ -130,6 +136,14 @@ export default class Live { }) } + /** @return the most important events, to be shown in the UI */ + private importantEvents() { + return this.events + .toArray() + .slice(0, this.opts.events || 8) + .sort((a, b) => a.timestamp - b.timestamp) + } + private readonly lookup: Record = {} /** Add `line` to our heap `this.events` */ private pushEvent(line: string, metric: WorkerState, timestamp: number) { @@ -163,16 +177,21 @@ export default class Live { if (this.opts.events === 0) { return [] } else { - return this.events - .toArray() - .slice(0, this.opts.events || 8) - .sort((a, b) => a.timestamp - b.timestamp) + return this.importantEvents() } } - /** `pushEvent` and then pass the updated model to `cb` */ - private pushEventAndPublish(line: string, metric: WorkerState, timestamp: number, cb: OnData) { - cb({ events: this.pushEvent(line, metric, timestamp), workers: Object.values(this.workers) }) + /** Helps with debouncing logLine updates */ + private logLineTO: null | ReturnType = null + + /** Add the given `line` to our logLines model and pass the updated model to `cb` */ + private pushLineAndPublish(logLine: string, cb: OnData) { + if (logLine) { + // here we avoid a flood of React renders by batching them up a + // bit; i thought react 18 was supposed to help with this. hmm. + if (this.logLineTO) clearTimeout(this.logLineTO) + this.logLineTO = setTimeout(() => cb({ logLine }), 1) + } } private asMillisSinceEpoch(timestamp: string) { diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/tailf.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/tailf.ts index 76fd93fd..b4fcae5d 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/tailf.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/tailf.ts @@ -19,9 +19,10 @@ import split2 from "split2" import chokidar from "chokidar" import TailFile from "@logdna/tail-file" -import Kind, { resourcePaths } from "./kinds.js" +import Kind, { KindedSource, resourcePaths } from "./kinds.js" export type Tail = { + kind: Kind stream: import("stream").Readable quit: TailFile["quit"] } @@ -34,7 +35,7 @@ export function waitTillExists(filepath: string) { }) } -async function initTail(filepath: string, split = true): Promise { +async function initTail({ kind, filepath }: KindedSource, split = true): Promise { await waitTillExists(filepath) return new Promise((resolve, reject) => { @@ -47,17 +48,23 @@ async function initTail(filepath: string, split = true): Promise { tail.start() resolve({ + kind, stream: split ? tail.pipe(split2()) : tail, quit: tail.quit.bind(tail), }) }) } -export async function pathsFor(kind: Kind, profile: string, jobId: string) { +export async function pathsFor(mkind: Kind, profile: string, jobId: string) { const { Profiles } = await import("madwizard") - return resourcePaths[kind].map((resourcePath) => - join(Profiles.guidebookJobDataPath({ profile }), jobId, resourcePath) - ) + return resourcePaths[mkind].map((src) => { + const kind = typeof src === "string" ? mkind : src.kind + const resourcePath = typeof src === "string" ? src : src.filepath + return { + kind, + filepath: join(Profiles.guidebookJobDataPath({ profile }), jobId, resourcePath), + } + }) } export default async function tailf( @@ -66,5 +73,5 @@ export default async function tailf( jobId: string, split = true ): Promise[]> { - return pathsFor(kind, profile, jobId).then((_) => _.map((filepath) => initTail(filepath, split))) + return pathsFor(kind, profile, jobId).then((_) => _.map((src) => initTail(src, split))) } diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dump.ts b/plugins/plugin-codeflare-dashboard/src/controller/dump.ts index a9bc4012..f742b419 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/dump.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/dump.ts @@ -17,7 +17,7 @@ import { Arguments } from "@kui-shell/core" import { pathsFor } from "./dashboard/tailf.js" -import { isValidKind } from "./dashboard/kinds.js" +import { filepathOf, isValidKind } from "./dashboard/kinds.js" import { Options as DashboardOptions, jobIdFrom, usage as dbUsage } from "./dashboard/index.js" export type Options = DashboardOptions & { @@ -50,7 +50,7 @@ export default async function dump(args: Arguments) { if (kind === "path") { // print the path to the data captured for the given jobId in the given profile - return import("./path.js").then((_) => _.pathFor(profile, jobId)) + return import("./path.js").then((_) => _.pathFor("env", profile, jobId)) } else if (kind === "env") { // print job env vars return JSON.stringify(await import("./env.js").then((_) => _.getJobEnv(profile, jobId)), undefined, 2) @@ -60,10 +60,10 @@ export default async function dump(args: Arguments) { ( await pathsFor(kind, profile, jobId) ).map( - (filepath) => + (src) => new Promise((resolve, reject) => { try { - const rs = createReadStream(filepath) + const rs = createReadStream(filepathOf(src)) rs.on("close", resolve) rs.pipe(process.stdout) } catch (err) { diff --git a/plugins/plugin-codeflare-dashboard/src/controller/env.ts b/plugins/plugin-codeflare-dashboard/src/controller/env.ts index c921c58b..db62a38a 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/env.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/env.ts @@ -16,6 +16,8 @@ import { join } from "path" +import { resourcePaths, filepathOf } from "./dashboard/kinds.js" + type NameValue = { name: string; value: unknown } function isNameValue(obj: object): obj is NameValue { @@ -36,7 +38,7 @@ function toRecord(nva: NameValue[]): Record { } async function getJobEnvFilepath(profile: string, jobId: string): Promise { - return join(await import("./path.js").then((_) => _.pathFor(profile, jobId)), "env.json") + return join(await import("./path.js").then((_) => _.pathFor("env", profile, jobId)), filepathOf(resourcePaths.env[0])) } export async function getJobEnv(profile: string, jobId: string): Promise> { diff --git a/plugins/plugin-codeflare-dashboard/src/controller/path.ts b/plugins/plugin-codeflare-dashboard/src/controller/path.ts index 29fcb15e..ec0f0e96 100644 --- a/plugins/plugin-codeflare-dashboard/src/controller/path.ts +++ b/plugins/plugin-codeflare-dashboard/src/controller/path.ts @@ -14,10 +14,11 @@ * limitations under the License. */ +import type Kind from "./dashboard/kinds.js" import { pathsFor } from "./dashboard/tailf.js" /** @return path to the data captured for the given jobId in the given profile */ -export async function pathFor(profile: string, jobId: string) { +export async function pathFor(kind: Kind, profile: string, jobId: string): Promise { const { dirname } = await import("path") - return Array.from(new Set(await pathsFor("cpu%", profile, jobId).then((_) => _.map((_) => dirname(dirname(_))))))[0] + return Array.from(new Set(await pathsFor(kind, profile, jobId).then((_) => _.map((_) => dirname(_.filepath)))))[0] }