From acd51c32d77b967e5b999e6d1528d7e8b0abe52c Mon Sep 17 00:00:00 2001 From: Nick Mitchell Date: Wed, 19 Apr 2023 12:19:46 -0400 Subject: [PATCH] fix: Top should allow uparrown/downarrow to cycle through namespaces also refactors top controller into its own subdir, and refactors it into smaller files in that dir --- .../src/components/Top/index.tsx | 86 +++- .../src/components/Top/types.ts | 6 + .../src/controller/dashboard/top.ts | 382 ------------------ .../src/controller/dashboard/top/index.ts | 55 +++ .../src/controller/dashboard/top/options.ts | 36 ++ .../src/controller/dashboard/top/parsers.ts | 78 ++++ .../src/controller/dashboard/top/stats.ts | 87 ++++ .../src/controller/dashboard/top/watcher.ts | 171 ++++++++ .../src/controller/kubernetes.ts | 68 ++++ .../plugin-codeflare-dashboard/src/plugin.ts | 4 +- 10 files changed, 577 insertions(+), 396 deletions(-) delete mode 100644 plugins/plugin-codeflare-dashboard/src/controller/dashboard/top.ts create mode 100644 plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/index.ts create mode 100644 plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/options.ts create mode 100644 plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/parsers.ts create mode 100644 plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/stats.ts create mode 100644 plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/watcher.ts create mode 100644 plugins/plugin-codeflare-dashboard/src/controller/kubernetes.ts diff --git a/plugins/plugin-codeflare-dashboard/src/components/Top/index.tsx b/plugins/plugin-codeflare-dashboard/src/components/Top/index.tsx index b178adf6..b6a9918f 100644 --- a/plugins/plugin-codeflare-dashboard/src/components/Top/index.tsx +++ b/plugins/plugin-codeflare-dashboard/src/components/Top/index.tsx @@ -20,7 +20,14 @@ import { emitKeypressEvents } from "readline" import { Box, Text, render } from "ink" import type Group from "./Group.js" -import type { OnData, UpdatePayload, ResourceSpec } from "./types.js" +import type { + Context, + ChangeContextRequest, + ChangeContextRequestHandler, + WatcherInitializer, + UpdatePayload, + ResourceSpec, +} from "./types.js" import JobBox from "./JobBox.js" import defaultValueFor from "./defaults.js" @@ -32,11 +39,19 @@ type UI = { refreshCycle?: number } -type Props = UI & { - initWatcher: (cb: OnData) => void -} +type Props = UI & + Context /* initial context */ & { + /** UI is ready to consume model updates */ + initWatcher: WatcherInitializer + + /** Ui wants to change context */ + changeContext: ChangeContextRequestHandler + } type State = UI & { + /** Current watcher */ + watcher: { kill(): void } + /** Model from controller */ rawModel: UpdatePayload @@ -63,8 +78,27 @@ class Top extends React.PureComponent { return ((n % d) + d) % d } - public componentDidMount() { - this.props.initWatcher(this.onData) + /** Do we have a selected group? */ + private get hasSelection() { + return this.state?.selectedGroupIdx >= 0 && this.state?.selectedGroupIdx < this.state.groups.length + } + + /** Current cluster context */ + private get currentContext() { + return { + cluster: this.state?.rawModel?.cluster || this.props.cluster, + namespace: this.state?.rawModel?.namespace || this.props.namespace, + } + } + + /** Updated cluster context */ + private updatedContext({ which }: Pick, next: string) { + return Object.assign(this.currentContext, which === "namespace" ? { namespace: next } : { cluster: next }) + } + + public async componentDidMount() { + this.setState({ watcher: await this.props.initWatcher(this.currentContext, this.onData) }) + this.initRefresher() this.initKeyboardEvents() } @@ -117,8 +151,23 @@ class Top extends React.PureComponent { case "escape": this.setState({ selectedGroupIdx: -1 }) break + case "up": + case "down": + /** Change context selection */ + if (this.state?.rawModel.namespace) { + this.props + .changeContext({ which: "namespace", from: this.state.rawModel.namespace, dir: key.name }) + .then((next) => { + if (next) { + this.reinit(this.updatedContext({ which: "namespace" }, next)) + } + }) + } + break + case "left": case "right": + /** Change job selection */ if (this.state.groups) { const incr = key.name === "left" ? -1 : 1 this.setState((curState) => ({ @@ -145,8 +194,25 @@ class Top extends React.PureComponent { }) } + private get emptyStats(): UpdatePayload["stats"] { + return { min: { cpu: 0, mem: 0, gpu: 0 }, tot: {} } + } + + private reinit(context: Context) { + if (this.state?.watcher) { + this.state?.watcher.kill() + } + this.setState({ groups: [], rawModel: Object.assign({ hosts: [], stats: this.emptyStats }, context) }) + this.props.initWatcher(context, this.onData) + } + /** We have received data from the controller */ - private readonly onData = (rawModel: UpdatePayload) => + private readonly onData = (rawModel: UpdatePayload) => { + if (rawModel.cluster !== this.currentContext.cluster || rawModel.namespace !== this.currentContext.namespace) { + // this is straggler data from the prior context + return + } + this.setState((curState) => { if (JSON.stringify(curState?.rawModel) === JSON.stringify(rawModel)) { return null @@ -154,6 +220,7 @@ class Top extends React.PureComponent { return { rawModel, groups: this.groupBy(rawModel) } } }) + } private groupBy(model: UpdatePayload): State["groups"] { return Object.values( @@ -192,11 +259,6 @@ class Top extends React.PureComponent { ) } - /** Do we have a selected group? */ - private get hasSelection() { - return this.state?.selectedGroupIdx >= 0 && this.state?.selectedGroupIdx < this.state.groups.length - } - private mostOf({ request, limit }: ResourceSpec, defaultValue: number) { if (request === -1 && limit === -1) { return defaultValue diff --git a/plugins/plugin-codeflare-dashboard/src/components/Top/types.ts b/plugins/plugin-codeflare-dashboard/src/components/Top/types.ts index 69c7f85c..4de03c66 100644 --- a/plugins/plugin-codeflare-dashboard/src/components/Top/types.ts +++ b/plugins/plugin-codeflare-dashboard/src/components/Top/types.ts @@ -65,3 +65,9 @@ export type Context = { export type UpdatePayload = Context & JobsByHost export type OnData = (payload: UpdatePayload) => void + +export type WatcherInitializer = (context: Context, cb: OnData) => Promise<{ kill(): void }> + +export type ChangeContextRequest = { which: "context" | "namespace"; from: string; dir: "down" | "up" } + +export type ChangeContextRequestHandler = (req: ChangeContextRequest) => Promise diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top.ts deleted file mode 100644 index 6800a1fb..00000000 --- a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top.ts +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright 2023 The Kubernetes Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import Debug from "debug" -import type { Arguments } from "@kui-shell/core" - -import type Options from "./job/options.js" -import type { OnData, HostRec, PodRec, ResourceSpec, UpdatePayload } from "../../components/Top/types.js" - -import { enterAltBufferMode } from "./term.js" -import defaultValueFor from "../../components/Top/defaults.js" - -export type MyOptions = Options & { - /** Don't show job names in the UI */ - redact: boolean - - /** Show just my jobs */ - m: boolean - me: boolean - - /** Show jobs for one namespace */ - n: string - namespace: string - - /** Show jobs across all namespaces */ - A: boolean - "all-namespaces": boolean -} - -/** @return as milli cpus, or -1 if not specified */ -function parseCpu(amount: string): number { - try { - if (amount === "") { - return -1 - } else if (amount[amount.length - 1] === "m") { - return parseInt(amount.slice(0, amount.length - 1), 10) - } else { - return parseInt(amount, 10) * 1000 - } - } catch (err) { - console.error("Odd cpu spec " + amount) - return defaultValueFor.cpu - } -} - -const unit = { - K: 1000, - Ki: 1024, - M: Math.pow(1000, 2), - Mi: Math.pow(1024, 2), - G: Math.pow(1000, 3), - Gi: Math.pow(1024, 3), - T: Math.pow(1000, 4), - Ti: Math.pow(1024, 4), - P: Math.pow(1000, 5), - Pi: Math.pow(1024, 5), - E: Math.pow(1000, 6), - Ei: Math.pow(1024, 6), -} - -type ValidUnit = keyof typeof unit - -function isValidUnit(u: string): u is ValidUnit { - return unit[u as ValidUnit] !== undefined -} - -/** @return as bytes, or -1 if not specified */ -function parseMem(amount: string): number { - if (amount === "") { - return -1 - } else { - const match = amount.match(/^(\d+)((k|M|G|T|P|E)?i?)$/) - if (!match || (match[2] && !isValidUnit(match[2]))) { - console.error("Odd memory spec " + amount) - return -1 - } else { - return parseInt(match[1], 10) * (!match[2] ? 1 : unit[match[2] as ValidUnit]) - } - } -} - -/** @return as count, or 0 if not specified */ -function parseGpu(amount: string): number { - if (amount === "") { - return 0 - } else { - return parseInt(amount, 10) - } -} - -function leastOf({ request, limit }: ResourceSpec, defaultValue: number): number { - if (request === -1 && limit === -1) { - return defaultValue - } else if (request === -1) { - return limit - } else if (limit === -1) { - return request - } else { - return Math.min(request, limit) - } -} - -/** Map from host to map from jobname to pods */ -type Model = Record>> -// host job name - -async function getCurrentCluster(): Promise { - const { execFile } = await import("child_process") - return new Promise((resolve, reject) => { - try { - execFile("kubectl", ["config", "view", "--minify", "-o=jsonpath={.clusters[0].name}"], (err, stdout, stderr) => { - if (err) { - console.error(stderr) - reject(err) - } else { - // trim off port - resolve(stdout.replace(/:\d+$/, "")) - } - }) - } catch (err) { - reject(err) - } - }) -} - -async function getCurrentNamespace(): Promise { - const { execFile } = await import("child_process") - return new Promise((resolve, reject) => { - try { - execFile("kubectl", ["config", "view", "--minify", "-o=jsonpath={..namespace}"], (err, stdout, stderr) => { - if (err) { - console.error(stderr) - reject(err) - } else { - resolve(stdout) - } - }) - } catch (err) { - reject(err) - } - }) -} - -async function getNamespaceFromArgsOrCurrent(args: Arguments) { - const namespace = args.parsedOptions.A ? "All Namespaces" : args.parsedOptions.n || (await getCurrentNamespace()) - const namespaceCommandLineOption = args.parsedOptions.A - ? "--all-namespaces" - : args.parsedOptions.n - ? `-n ${args.parsedOptions.n}` - : "" - - return { namespace, namespaceCommandLineOption } -} - -export default async function jobsController(args: Arguments) { - const debug = Debug("plugin-codeflare-dashboard/controller/top") - - if (process.env.ALT !== "false") { - enterAltBufferMode() - } - - // To help us parse out one "record's" worth of output from kubectl - const recordSeparator = "-----------" - - const [cluster, { namespace, namespaceCommandLineOption }] = await Promise.all([ - getCurrentCluster(), - getNamespaceFromArgsOrCurrent(args), - ]) - debug("cluster", cluster) - debug("namespace", namespace || "using namespace from user current context") - - debug("spawning watcher...") - const { spawn } = await import("child_process") - const child = spawn( - "bash", - [ - "-c", - `"while true; do kubectl get pod ${namespaceCommandLineOption} --no-headers -o=custom-columns=NAME:.metadata.name,JOB:'.metadata.labels.app\\.kubernetes\\.io/instance',HOST:.status.hostIP,CPU:'.spec.containers[0].resources.requests.cpu',CPUL:'.spec.containers[0].resources.limits.cpu',MEM:'.spec.containers[0].resources.requests.memory',MEML:'.spec.containers[0].resources.limits.memory',GPU:.spec.containers[0].resources.requests.'nvidia\\.com/gpu',GPUL:.spec.containers[0].resources.limits.'nvidia\\.com/gpu',JOB2:'.metadata.labels.appwrapper\\.mcad\\.ibm\\.com',CTIME:.metadata.creationTimestamp,USER:'.metadata.labels.app\\.kubernetes\\.io/owner'; echo '${recordSeparator}'; sleep 2; done"`, - ], - { shell: "/bin/bash", stdio: ["ignore", "pipe", "inherit"] } - ) - debug("spawned watcher") - - const jobIndices: Record = {} // lookup - const jobOcc: (undefined | string)[] = [] // occupancy vector - const jobIdxFor = (job: string): number => { - const jobIdx = jobIndices[job] - if (jobIdx !== undefined) { - return jobIdx - } else { - for (let idx = 0; idx < jobOcc.length; idx++) { - if (jobOcc[idx] === undefined) { - jobOcc[idx] = job - jobIndices[job] = idx - return idx - } - } - - const jobIdx = jobOcc.push(job) - 1 - jobIndices[job] = jobIdx - return jobIdx - } - } - const removeJobIdx = (job: string) => { - const jobIdx = jobIndices[job] - if (jobIdx !== undefined) { - delete jobIndices[job] - jobOcc[jobIdx] = undefined - } - } - - const trim = (extantJobs: Record) => { - Object.keys(jobIndices) - .filter((job) => !(job in extantJobs)) - .forEach(removeJobIdx) - } - - const initWatcher = (cb: OnData) => { - debug("init watcher callbacks") - const me = process.env.USER || "NOUSER" - - child.on("error", (err) => console.error(err)) - - let leftover = "" - child.stdout.on("data", (data) => { - const sofar = leftover + data.toString() - - const term = sofar.indexOf(recordSeparator) - if (term < 0) { - leftover = sofar - } else if (term >= 0) { - leftover = sofar.slice(term + recordSeparator.length) - - const lines = sofar.slice(0, term).split(/\n/).filter(Boolean) - if (lines.length === 0) { - return - } - - const byHost: Model = lines - .map((_) => _.split(/\s+/)) - .map((A) => ({ - name: A[0], - job: A[1] === "" ? A[9] : A[1], - host: A[2], - ctime: A[10] === "" ? Date.now() : new Date(A[10]).getTime(), - owner: A[11], - cpu: { request: parseCpu(A[3]), limit: parseCpu(A[4]) }, - mem: { request: parseMem(A[5]), limit: parseMem(A[6]) }, - gpu: { request: parseGpu(A[7]), limit: parseGpu(A[8]) }, - })) - .filter((_) => _.job && _.job !== "") // exclude pods not associated with a job - .filter((_) => !args.parsedOptions.me || _.owner === me) // exclude pods not owned by me? - .map((rec) => - Object.assign(rec, { - jobIdx: jobIdxFor(rec.job), - tot: { - cpu: leastOf(rec.cpu, defaultValueFor.cpu), - mem: leastOf(rec.mem, defaultValueFor.mem), - gpu: leastOf(rec.gpu, defaultValueFor.gpu), - }, - }) - ) - .reduce((byHost, rec) => { - // pod is not yet mapped to a host? - if (rec.host !== "") { - if (!byHost[rec.host]) { - byHost[rec.host] = {} - } - const byJob = byHost[rec.host] - - if (!byJob[rec.job]) { - byJob[rec.job] = {} - } - const byName = byJob[rec.job] - - byName[rec.name] = rec - } - - return byHost - }, {} as Model) - - const extantJobs = Object.values(byHost) - .flatMap((byJob) => Object.keys(byJob)) - .reduce((jobs, job) => { - jobs[job] = true - return jobs - }, {} as Record) - trim(extantJobs) - - // turn the records of records into arrays to make the UI code - // cleaner - const hosts = Object.keys(byHost).map((host) => ({ - host, - jobs: Object.keys(byHost[host] || []).map((name) => ({ - name: args.parsedOptions.redact ? `Job ${jobIdxFor(name)}` : name, - jobIdx: jobIdxFor(name), - pods: Object.values(byHost[host][name] || []), - })), - })) - - cb(Object.assign({ cluster, namespace }, stats(hosts))) - } - }) - } - - debug("loading UI dependencies") - const [{ default: render }] = await Promise.all([import("../../components/Top/index.js")]) - - debug("rendering") - await render({ initWatcher }) - debug("exiting") - return true -} - -/** @return greatest common divisor of `a` and `b` */ -//const gcd = (a: number, b: number): number => a ? gcd(b % a, a) : b -function gcd(a: number, b: number) { - a = Math.abs(a) - b = Math.abs(b) - if (b > a) { - const temp = a - a = b - b = temp - } - // eslint-disable-next-line no-constant-condition - while (true) { - if (b == 0) return a - a %= b - if (a == 0) return b - b %= a - } -} - -/** Extract min/total values for resource demand */ -function stats(hosts: HostRec[]): Pick { - // find the min cpu, total cpu, etc. - const stats = hosts - .flatMap((_) => _.jobs.flatMap((_) => _.pods)) - .reduce( - (stats, pod) => { - const cpu = leastOf(pod.cpu, defaultValueFor.cpu) - const mem = leastOf(pod.mem, defaultValueFor.mem) - const gpu = leastOf(pod.gpu, defaultValueFor.gpu) - - stats.min.cpu = stats.min.cpu === Number.MAX_VALUE ? cpu : gcd(stats.min.cpu, cpu) - //stats.min.mem = stats.min.mem === Number.MAX_VALUE ? mem / 1024 / 1024 : gcd(stats.min.mem, mem / 1024 / 1024) - stats.min.mem = Math.min(stats.min.mem, mem) - stats.min.gpu = Math.min(stats.min.gpu, gpu) - - stats.tot[pod.host].cpu += cpu - stats.tot[pod.host].mem += mem - stats.tot[pod.host].gpu += gpu - - return stats - }, - { - min: { cpu: Number.MAX_VALUE, mem: 32 * unit.Gi, gpu: Number.MAX_VALUE }, - tot: hosts.reduce((T, host) => { - T[host.host] = { cpu: 0, mem: 0, gpu: 0 } - return T - }, {} as UpdatePayload["stats"]["tot"]), - } as UpdatePayload["stats"] - ) - - return { - hosts, - stats, - } -} diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/index.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/index.ts new file mode 100644 index 00000000..86ca2589 --- /dev/null +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/index.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2023 The Kubernetes Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Debug from "debug" + +import type { Arguments } from "@kui-shell/core" +import type TopOptions from "./options.js" + +import { enterAltBufferMode } from "../term.js" +import { getCurrentCluster, getCurrentNamespace, changeContext } from "../../kubernetes.js" + +import initWatcher from "./watcher.js" + +export async function getNamespaceFromArgsOrCurrent(args: Arguments) { + return /*args.parsedOptions.A ? "All Namespaces" :*/ args.parsedOptions.n || (await getCurrentNamespace()) +} + +export default async function jobsController(args: Arguments) { + const debug = Debug("plugin-codeflare-dashboard/controller/top") + + if (process.env.ALT !== "false") { + enterAltBufferMode() + } + + // these will be the initial values of cluster and namespace focus + const [cluster, ns] = await Promise.all([getCurrentCluster(), getNamespaceFromArgsOrCurrent(args)]) + debug("cluster", cluster) + debug("namespace", ns || "using namespace from user current context") + + debug("loading UI dependencies") + const [{ default: render }] = await Promise.all([import("../../../components/Top/index.js")]) + + debug("rendering") + await render({ + cluster, + namespace: ns, + initWatcher: initWatcher.bind(args.parsedOptions), + changeContext, + }) + debug("exiting") + return true +} diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/options.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/options.ts new file mode 100644 index 00000000..5619d457 --- /dev/null +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/options.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2023 The Kubernetes Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type Options from "../job/options.js" + +type TopOptions = Options & { + /** Don't show job names in the UI */ + redact: boolean + + /** Show just my jobs */ + m: boolean + me: boolean + + /** Show jobs for one namespace */ + n: string + namespace: string + + /** Show jobs across all namespaces */ + A: boolean + "all-namespaces": boolean +} + +export default TopOptions diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/parsers.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/parsers.ts new file mode 100644 index 00000000..f324598b --- /dev/null +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/parsers.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2023 The Kubernetes Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import defaultValueFor from "../../../components/Top/defaults.js" + +/** @return as milli cpus, or -1 if not specified */ +export function parseCpu(amount: string): number { + try { + if (amount === "") { + return -1 + } else if (amount[amount.length - 1] === "m") { + return parseInt(amount.slice(0, amount.length - 1), 10) + } else { + return parseInt(amount, 10) * 1000 + } + } catch (err) { + console.error("Odd cpu spec " + amount) + return defaultValueFor.cpu + } +} + +export const unit = { + K: 1000, + Ki: 1024, + M: Math.pow(1000, 2), + Mi: Math.pow(1024, 2), + G: Math.pow(1000, 3), + Gi: Math.pow(1024, 3), + T: Math.pow(1000, 4), + Ti: Math.pow(1024, 4), + P: Math.pow(1000, 5), + Pi: Math.pow(1024, 5), + E: Math.pow(1000, 6), + Ei: Math.pow(1024, 6), +} + +type ValidUnit = keyof typeof unit + +export function isValidUnit(u: string): u is ValidUnit { + return unit[u as ValidUnit] !== undefined +} + +/** @return as bytes, or -1 if not specified */ +export function parseMem(amount: string): number { + if (amount === "") { + return -1 + } else { + const match = amount.match(/^(\d+)((k|M|G|T|P|E)?i?)$/) + if (!match || (match[2] && !isValidUnit(match[2]))) { + console.error("Odd memory spec " + amount) + return -1 + } else { + return parseInt(match[1], 10) * (!match[2] ? 1 : unit[match[2] as ValidUnit]) + } + } +} + +/** @return as count, or 0 if not specified */ +export function parseGpu(amount: string): number { + if (amount === "") { + return 0 + } else { + return parseInt(amount, 10) + } +} diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/stats.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/stats.ts new file mode 100644 index 00000000..8b7be7f9 --- /dev/null +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/stats.ts @@ -0,0 +1,87 @@ +/* + * Copyright 2023 The Kubernetes Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { HostRec, ResourceSpec, UpdatePayload } from "../../../components/Top/types.js" + +import { unit } from "./parsers.js" +import defaultValueFor from "../../../components/Top/defaults.js" + +export function leastOf({ request, limit }: ResourceSpec, defaultValue: number): number { + if (request === -1 && limit === -1) { + return defaultValue + } else if (request === -1) { + return limit + } else if (limit === -1) { + return request + } else { + return Math.min(request, limit) + } +} + +/** @return greatest common divisor of `a` and `b` */ +function gcd(a: number, b: number) { + a = Math.abs(a) + b = Math.abs(b) + if (b > a) { + const temp = a + a = b + b = temp + } + // eslint-disable-next-line no-constant-condition + while (true) { + if (b == 0) return a + a %= b + if (a == 0) return b + b %= a + } +} + +/** Extract min/total values for resource demand */ +export function stats(hosts: HostRec[]): Pick { + // find the min cpu, total cpu, etc. + const stats = hosts + .flatMap((_) => _.jobs.flatMap((_) => _.pods)) + .reduce( + (stats, pod) => { + const cpu = leastOf(pod.cpu, defaultValueFor.cpu) + const mem = leastOf(pod.mem, defaultValueFor.mem) + const gpu = leastOf(pod.gpu, defaultValueFor.gpu) + + stats.min.cpu = stats.min.cpu === Number.MAX_VALUE ? cpu : gcd(stats.min.cpu, cpu) + //stats.min.mem = stats.min.mem === Number.MAX_VALUE ? mem / 1024 / 1024 : gcd(stats.min.mem, mem / 1024 / 1024) + stats.min.mem = Math.min(stats.min.mem, mem) + stats.min.gpu = Math.min(stats.min.gpu, gpu) + + stats.tot[pod.host].cpu += cpu + stats.tot[pod.host].mem += mem + stats.tot[pod.host].gpu += gpu + + return stats + }, + { + min: { cpu: Number.MAX_VALUE, mem: 32 * unit.Gi, gpu: Number.MAX_VALUE }, + tot: hosts.reduce((T, host) => { + T[host.host] = { cpu: 0, mem: 0, gpu: 0 } + return T + }, {} as UpdatePayload["stats"]["tot"]), + } as UpdatePayload["stats"] + ) + + return { + hosts, + stats, + } +} diff --git a/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/watcher.ts b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/watcher.ts new file mode 100644 index 00000000..52e70f20 --- /dev/null +++ b/plugins/plugin-codeflare-dashboard/src/controller/dashboard/top/watcher.ts @@ -0,0 +1,171 @@ +/* + * Copyright 2023 The Kubernetes Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Debug from "debug" + +import type TopOptions from "./options.js" +import type { Context, OnData, PodRec } from "../../../components/Top/types.js" + +import { leastOf, stats } from "./stats.js" +import { parseCpu, parseMem, parseGpu } from "./parsers.js" +import defaultValueFor from "../../../components/Top/defaults.js" + +/** Map from host to map from jobname to pods */ +type Model = Record>> +// host job name + +export default async function initWatcher(this: TopOptions, { cluster, namespace: ns }: Context, cb: OnData) { + const debug = Debug("plugin-codeflare-dashboard/controller/top") + debug("init watcher callbacks", cluster, ns) + + // To help us parse out one "record's" worth of output from kubectl + const recordSeparator = "-----------" + + const jobIndices: Record = {} // lookup + const jobOcc: (undefined | string)[] = [] // occupancy vector + const jobIdxFor = (job: string): number => { + const jobIdx = jobIndices[job] + if (jobIdx !== undefined) { + return jobIdx + } else { + for (let idx = 0; idx < jobOcc.length; idx++) { + if (jobOcc[idx] === undefined) { + jobOcc[idx] = job + jobIndices[job] = idx + return idx + } + } + + const jobIdx = jobOcc.push(job) - 1 + jobIndices[job] = jobIdx + return jobIdx + } + } + const removeJobIdx = (job: string) => { + const jobIdx = jobIndices[job] + if (jobIdx !== undefined) { + delete jobIndices[job] + jobOcc[jobIdx] = undefined + } + } + + const trim = (extantJobs: Record) => { + Object.keys(jobIndices) + .filter((job) => !(job in extantJobs)) + .forEach(removeJobIdx) + } + + const me = process.env.USER || "NOUSER" + + debug("spawning watcher...") + const { spawn } = await import("child_process") + const child = spawn( + "bash", + [ + "-c", + `"while true; do kubectl get pod -n ${ns} --no-headers -o=custom-columns=NAME:.metadata.name,JOB:'.metadata.labels.app\\.kubernetes\\.io/instance',HOST:.status.hostIP,CPU:'.spec.containers[0].resources.requests.cpu',CPUL:'.spec.containers[0].resources.limits.cpu',MEM:'.spec.containers[0].resources.requests.memory',MEML:'.spec.containers[0].resources.limits.memory',GPU:.spec.containers[0].resources.requests.'nvidia\\.com/gpu',GPUL:.spec.containers[0].resources.limits.'nvidia\\.com/gpu',JOB2:'.metadata.labels.appwrapper\\.mcad\\.ibm\\.com',CTIME:.metadata.creationTimestamp,USER:'.metadata.labels.app\\.kubernetes\\.io/owner'; echo '${recordSeparator}'; sleep 2; done"`, + ], + { shell: "/bin/bash", stdio: ["ignore", "pipe", "inherit"] } + ) + debug("spawned watcher") + process.on("exit", () => child.kill()) + + child.on("error", (err) => console.error(err)) + child.on("exit", (code) => debug("watcher subprocess exiting", code)) + + let leftover = "" + child.stdout.on("data", (data) => { + const sofar = leftover + data.toString() + + const term = sofar.indexOf(recordSeparator) + if (term < 0) { + leftover = sofar + } else if (term >= 0) { + leftover = sofar.slice(term + recordSeparator.length) + + const lines = sofar.slice(0, term).split(/\n/).filter(Boolean) + if (lines.length === 0) { + return + } + + const byHost: Model = lines + .map((_) => _.split(/\s+/)) + .map((A) => ({ + name: A[0], + job: A[1] === "" ? A[9] : A[1], + host: A[2], + ctime: A[10] === "" ? Date.now() : new Date(A[10]).getTime(), + owner: A[11], + cpu: { request: parseCpu(A[3]), limit: parseCpu(A[4]) }, + mem: { request: parseMem(A[5]), limit: parseMem(A[6]) }, + gpu: { request: parseGpu(A[7]), limit: parseGpu(A[8]) }, + })) + .filter((_) => _.job && _.job !== "") // exclude pods not associated with a job + .filter((_) => !this.me || _.owner === me) // exclude pods not owned by me? + .map((rec) => + Object.assign(rec, { + jobIdx: jobIdxFor(rec.job), + tot: { + cpu: leastOf(rec.cpu, defaultValueFor.cpu), + mem: leastOf(rec.mem, defaultValueFor.mem), + gpu: leastOf(rec.gpu, defaultValueFor.gpu), + }, + }) + ) + .reduce((byHost, rec) => { + // pod is not yet mapped to a host? + if (rec.host !== "") { + if (!byHost[rec.host]) { + byHost[rec.host] = {} + } + const byJob = byHost[rec.host] + + if (!byJob[rec.job]) { + byJob[rec.job] = {} + } + const byName = byJob[rec.job] + + byName[rec.name] = rec + } + + return byHost + }, {} as Model) + + const extantJobs = Object.values(byHost) + .flatMap((byJob) => Object.keys(byJob)) + .reduce((jobs, job) => { + jobs[job] = true + return jobs + }, {} as Record) + trim(extantJobs) + + // turn the records of records into arrays to make the UI code + // cleaner + const hosts = Object.keys(byHost).map((host) => ({ + host, + jobs: Object.keys(byHost[host] || []).map((name) => ({ + name: this.redact ? `Job ${jobIdxFor(name)}` : name, + jobIdx: jobIdxFor(name), + pods: Object.values(byHost[host][name] || []), + })), + })) + + cb(Object.assign({ cluster, namespace: ns }, stats(hosts))) + } + }) + + return child +} diff --git a/plugins/plugin-codeflare-dashboard/src/controller/kubernetes.ts b/plugins/plugin-codeflare-dashboard/src/controller/kubernetes.ts new file mode 100644 index 00000000..d478b29b --- /dev/null +++ b/plugins/plugin-codeflare-dashboard/src/controller/kubernetes.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2023 The Kubernetes Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ChangeContextRequest } from "../components/Top/types.js" + +export async function kubectl(argv: string[], kubectl = "kubectl", quiet = false): Promise { + const { execFile } = await import("child_process") + return new Promise((resolve, reject) => { + try { + execFile(kubectl, argv, (err, stdout, stderr) => { + if (err) { + if (!quiet) { + console.error(stderr) + } + reject(err) + } else { + // trim off port + resolve(stdout) + } + }) + } catch (err) { + reject(err) + } + }) +} + +export async function getCurrentCluster(): Promise { + return kubectl(["config", "view", "--minify", "-o=jsonpath={.clusters[0].name}"]).then((_) => _.replace(/:\d+$/, "")) +} + +export async function getCurrentNamespace(): Promise { + return kubectl(["config", "view", "--minify", "-o=jsonpath={..namespace}"]) +} + +function listNamespaces(): Promise { + return kubectl(["get", "ns", "-o=name"], undefined, true) + .catch(() => kubectl(["projects", "-q"], "oc")) + .then((raw) => raw.split(/\n/)) + .then((_) => _.map((ns) => ns.replace(/^namespace\//, ""))) +} + +function mod(n: number, d: number) { + return ((n % d) + d) % d +} + +export async function changeContext(req: ChangeContextRequest) { + if (req.which === "namespace") { + const namespaces = await listNamespaces() + const idx = namespaces.indexOf(req.from) + if (idx >= 0) { + const newIdx = mod(idx + (req.dir === "up" ? -1 : 1), namespaces.length - 1) + return namespaces[newIdx] + } + } +} diff --git a/plugins/plugin-codeflare-dashboard/src/plugin.ts b/plugins/plugin-codeflare-dashboard/src/plugin.ts index a79c39c8..1090f47c 100644 --- a/plugins/plugin-codeflare-dashboard/src/plugin.ts +++ b/plugins/plugin-codeflare-dashboard/src/plugin.ts @@ -15,7 +15,7 @@ */ import type { KResponse, Registrar } from "@kui-shell/core" -import type { MyOptions as TopOptions } from "./controller/dashboard/top.js" +import type TopOptions from "./controller/dashboard/top/options.js" import type DashboardOptions from "./controller/dashboard/job/options.js" import { flags } from "./controller/dashboard/options.js" @@ -31,7 +31,7 @@ export default function registerCodeflareCommands(registrar: Registrar) { registrar.listen( `/codeflare/top`, - (args) => import("./controller/dashboard/top.js").then((_) => _.default(args)), + (args) => import("./controller/dashboard/top/index.js").then((_) => _.default(args)), { flags } )