initial anki commit

package.json

@@ -1,12 +1,33 @@
 {
   "name": "noob_agent",
   "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
+  "description": "Agent for some local tooling I use",
+  "main": "dist/bundle.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "build": "webpack --mode development",
+    "release": "webpack --mode production",
+    "bs": "yarn build && yarn start",
+    "start": "node ./dist/bundle.js"
   },
   "keywords": [],
-  "author": "",
-  "license": "ISC"
+  "author": "Noobgam <[email protected]>",
+  "license": "ISC",
+  "repository": "[email protected]:Noobgam/noob-agent.git",
+  "private": true,
+  "dependencies": {
+    "@types/node": "^20.4.5",
+    "express": "^4.18.2",
+    "node-fetch": "^3.3.2",
+    "prom-client": "^14.2.0",
+    "tslog": "^4.8.2",
+    "typescript": "^5.1.6"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.17",
+    "ts-loader": "^9.4.4",
+    "ts-node": "^10.9.1",
+    "webpack": "^5.88.2",
+    "webpack-cli": "^5.1.4",
+    "webpack-node-externals": "^3.0.0"
+  }
 }

src/anki/anki_config.ts

@@ -0,0 +1,4 @@
+export interface AnkiConfig {
+    // anki expected localhost, so no authorization
+    url: string;
+}

src/anki/client.ts

@@ -0,0 +1,69 @@
+import {AnkiConfig} from "./anki_config";
+import fetch from "node-fetch";
+
+interface AnkiConnectRequest {
+    action: string;
+    params?: object;
+}
+
+interface NoteInfo {
+    cards: number[],
+    fields: {
+        noteId: number;
+    },
+    tags: string[];
+}
+
+export class AnkiClient {
+    config: AnkiConfig;
+
+    constructor(config: AnkiConfig) {
+        this.config = config;
+    }
+
+    async ankiRequest(config: AnkiConfig, request: AnkiConnectRequest) {
+        const res = await fetch(config.url, {
+            method: 'POST',
+            body: JSON.stringify({
+                ...request,
+                version: 6,
+            }),
+        });
+        return await res.json();
+    }
+
+    async getDeckNames(): Promise<{ result: string[] }> {
+        return this.ankiRequest(this.config, {
+            action: 'deckNames',
+        }).then(data => data as ({ result : string[] }))
+    }
+
+    // (reviewTime, cardID, usn, buttonPressed, newInterval, previousInterval, newFactor, reviewDuration, reviewType)
+    async getDeckReviews(deckName: string, startId?: number): Promise<{ result: number[][] }> {
+        return this.ankiRequest(this.config, {
+            action: 'cardReviews',
+            params: {
+                deck: deckName,
+                startID: startId ?? 0,
+            }
+        }).then(data => data as ({ result : number[][] }))
+    }
+
+    async cardsToNotes(cards: number[]): Promise<{ result: number[]}> {
+        return this.ankiRequest(this.config, {
+            action: 'cardsToNotes',
+            params: {
+                cards: cards,
+            }
+        }).then(data => data as ({ result : number[] }))
+    }
+
+    async notesInfo(notes: number[]): Promise<{ result: NoteInfo[] }> {
+        return this.ankiRequest(this.config, {
+            action: 'notesInfo',
+            params: {
+                notes: notes,
+            }
+        }).then(data => data as ({ result : NoteInfo[] }))
+    }
+}

src/anki/collect_snapshot.ts

@@ -0,0 +1,47 @@
+import {metrics} from "../prometheus/metrics";
+import {AnkiClient} from "./client";
+import {log} from "../config";
+
+export async function collectSnapshot(anki: AnkiClient) {
+    const decks = (await anki.getDeckNames()).result;
+    const cardsToFetch: number[] = [];
+    log.info(`Fetching deck cards`);
+    for (const deck of decks) {
+        const deckResult = (await anki.getDeckReviews(deck)).result;
+        cardsToFetch.push(...deckResult.flatMap(d => d[1]))
+    }
+    const noteIds = (await anki.cardsToNotes(cardsToFetch)).result;
+    log.info(`Fetching ${noteIds.length} notes`);
+    // @ts-ignore
+    const allNotes = (await anki.notesInfo(noteIds)).result;
+    log.info(`Done fetching notes`);
+    if (metrics.locked) {
+        log.error("Cannot update metrics when locked. Discarding result.");
+        return;
+    }
+    metrics.locked = true;
+    try {
+        metrics.ankiReviewGauge.reset()
+        for (const deck of decks) {
+            const rawResult = await anki.getDeckReviews(deck);
+            for (let tuple of rawResult.result) {
+                const cardId = tuple[1];
+                const note = allNotes.find(note => note.cards.indexOf(cardId) !== -1);
+                if (!note) {
+                    log.error(`Could not match card ${cardId}`);
+                    continue;
+                }
+                const languages =
+                    note.tags.filter(t => t.startsWith('language_'))
+                        .map(t => t.substring('language_'.length));
+                if (languages.length != 1) {
+                    // untracked, this is something odd
+                    continue;
+                }
+                metrics.ankiReviewGauge.inc({deck_name: deck, language: languages[0]}, 1);
+            }
+        }
+    } finally {
+        metrics.locked = false;
+    }
+}

src/config.ts

@@ -0,0 +1,32 @@
+import {AnkiConfig} from "./anki/anki_config";
+import {PrometheusConfig} from "./prometheus/config";
+import {ILogObj, Logger} from "tslog";
+
+export interface Config {
+    prometheus: PrometheusConfig;
+    anki: AnkiConfig;
+}
+
+function requiredEnv(name: string): string {
+    const res = process.env[name];
+    if (!res) {
+        throw Error(`Environment variable ${name} is not defined`);
+    }
+    return res;
+}
+
+export const configureFromEnvironment: () => Config = () => {
+    return {
+        prometheus: {
+            url: requiredEnv("NOOBGAM_PROMETHEUS_PUSH_URL"),
+            username: requiredEnv("NOOBGAM_PROMETHEUS_USERNAME"),
+            password: requiredEnv("NOOBGAM_PROMETHEUS_PASSWORD"),
+        },
+        anki: {
+            url: 'http://127.0.0.1:8765',
+        }
+    }
+}
+
+export const log: Logger<ILogObj> = new Logger();
+export const globalConfig: Config = configureFromEnvironment();

src/index.ts

@@ -0,0 +1,35 @@
+import {globalConfig} from './config'
+import express from 'express'
+import {AnkiClient} from "./anki/client";
+import {pushMetrics} from "./prometheus/client";
+import {collectSnapshot} from "./anki/collect_snapshot";
+import {noopConcurrentInterval} from "./utils/functional";
+
+const app = express()
+const port = 3000
+
+app.listen(port, () => {
+    const anki = new AnkiClient(globalConfig.anki);
+
+    noopConcurrentInterval(
+        'refreshAnkiMetrics',
+        async () => {
+            try {
+                await collectSnapshot(anki);
+            } catch (e) {
+                console.log(e);
+            }
+        }, 30000
+    )
+
+    noopConcurrentInterval(
+        'prometheusPush',
+        async () => {
+            try {
+                await pushMetrics(globalConfig.prometheus);
+            } catch (e) {
+                console.log(e);
+            }
+        }, 5000
+    )
+})

src/prometheus/client.ts

@@ -0,0 +1,27 @@
+import {Pushgateway} from 'prom-client';
+import {PrometheusConfig} from "./config";
+import {metrics} from "./metrics";
+import {log} from "../config";
+
+export async function pushMetrics(config: PrometheusConfig) {
+    const client = new Pushgateway(
+        config.url,
+        {
+            headers: {
+                Authorization: `Basic ${btoa(config.username + ':' + config.password)})`
+            }
+        }
+    )
+    if (metrics.locked) {
+        // TODO: this is definitely yikes.
+        log.warn("Cannot publish metrics when locked. Discarding result.");
+        return;
+    }
+    metrics.locked = true;
+    try {
+        log.info(`Pushing metrics`);
+        return await client.push({jobName: 'anki_personal_monitoring'});
+    } finally {
+        metrics.locked = false
+    }
+}

src/prometheus/config.ts

@@ -0,0 +1,5 @@
+export interface PrometheusConfig {
+    url: string;
+    username: string;
+    password: string;
+}

src/prometheus/metrics.ts

@@ -0,0 +1,10 @@
+import {Gauge} from "prom-client";
+
+export const metrics = {
+    ankiReviewGauge: new Gauge({
+        name: 'anki_review',
+        help: 'Individual anki card review',
+        labelNames: ['deck_name', 'language']
+    }),
+    locked: false,
+}

src/utils/functional.ts

@@ -0,0 +1,20 @@
+import {log} from "../config";
+
+export function noopConcurrentInterval(name: string, callable: () => Promise<void>, intervalMs: number) {
+    let running = false;
+    setInterval(
+        async () => {
+            if (running) {
+                log.info(`Rejecting execution because [${name}] is already running`)
+                return;
+            }
+            try {
+                running = true;
+                await callable();
+            } finally {
+                running = false;
+            }
+        },
+        intervalMs
+    )
+}