Updates in packages

README.md

@@ -12,7 +12,6 @@ An api to query local language models using different backends
 - [Llama.cpp](https://github.com/ggerganov/llama.cpp/tree/master/examples/server)
 - [Koboldcpp](https://github.com/LostRuins/koboldcpp)
 - [Ollama](https://github.com/jmorganca/ollama)
-- [Goinfer](https://github.com/synw/goinfer)
 
 ## Quickstart
 

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@locallm/api",
-  "version": "0.0.29",
+  "version": "0.0.30",
   "description": "An api to query local language models using different backends",
   "repository": "https://github.com/synw/locallm",
   "scripts": {
@@ -12,7 +12,7 @@
     "restmix": "^0.4.0"
   },
   "devDependencies": {
-    "@locallm/types": "^0.0.14",
+    "@locallm/types": "^0.0.15",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-terser": "^0.4.4",
     "@rollup/plugin-typescript": "^11.1.5",

packages/api/src/providers/koboldcpp.ts

@@ -146,18 +146,18 @@ class KoboldcppProvider implements LmProvider {
     if (this.apiKey.length > 0) {
       headers["Authorization"] = `Bearer ${this.apiKey}`
     }
-    const response = await fetch(url, {
-      method: 'POST',
-      headers: headers,
-      body: body,
-    });
 
-    if (!response.body) {
-      throw new Error("No response body")
-    }
     let text = "";
     let data = {};
     if (inferenceParams?.stream == true) {
+      const response = await fetch(url, {
+        method: 'POST',
+        headers: headers,
+        body: body,
+      });
+      if (!response.body) {
+        throw new Error("No response body")
+      }
       let i = 1;
       let buf = new Array<string>();
       const eventStream = response.body

packages/api/src/providers/ollama.ts

@@ -1,4 +1,7 @@
 import { useApi } from "restmix";
+import { type ParsedEvent } from 'eventsource-parser'
+// @ts-ignore
+import { EventSourceParserStream } from 'eventsource-parser/stream';
 import { InferenceParams, InferenceResult, LmProvider, LmProviderParams, ModelConf } from "@locallm/types";
 //import { InferenceParams, InferenceResult, LmProvider, LmProviderParams, ModelConf } from "@/packages/types/interfaces.js";
 import { parseJson as parseJsonUtil } from './utils';
@@ -49,7 +52,11 @@ class OllamaProvider implements LmProvider {
     if (res.ok) {
       //console.log("RES", res.data);
       for (const m of res.data["models"]) {
-        this.models.push({ name: m.name, ctx: 2048 });
+        const info = {
+          size: m.details.parameter_size,
+          quant: m.details.quantization_level,
+        }
+        this.models.push({ name: m.name, ctx: -1, info: info });
       }
     } else {
       throw new Error(`Error ${res.status} loading models ${res.text}`);
@@ -140,6 +147,7 @@ class OllamaProvider implements LmProvider {
       raw = params.extra.raw;
       delete params.extra.raw;
     }
+    console.log("PARAMS", params);
     let inferParams: Record<string, any> = {
       model: this.model.name,
       prompt: prompt,
@@ -150,44 +158,45 @@ class OllamaProvider implements LmProvider {
       },
       ...params.extra
     }
-    if (params.threads) {
+    if (params.threads !== undefined) {
       inferParams.options.num_thread = params.threads;
     }
-    if (params.gpu_layers) {
+    if (params.gpu_layers !== undefined) {
       inferParams.options.gpu_layers = params.gpu_layers;
     }
-    if (params.repeat_penalty) {
+    if (params.repeat_penalty !== undefined) {
       inferParams.options.repeat_penalty = params.repeat_penalty;
     }
-    if (params.stop && params.stop.length > 0) {
+    if (params.stop !== undefined && params.stop?.length > 0) {
       inferParams.options.stop = params.stop;
     }
-    if (params.temperature) {
+    if (params.temperature !== undefined) {
       inferParams.options.temperature = params.temperature;
     }
-    if (params.tfs) {
+    if (params.tfs !== undefined) {
       inferParams.options.tfs_z = params.tfs;
     }
-    if (params.top_k) {
+    if (params.top_k !== undefined) {
       inferParams.options.top_k = params.top_k;
     }
-    if (params.top_p) {
+    if (params.top_p !== undefined) {
       inferParams.options.top_p = params.top_p;
     }
-    if (params.max_tokens) {
+    if (params.max_tokens !== undefined) {
       inferParams.options.num_predict = params.max_tokens;
     }
-    if (params.extra?.format) {
+    if (params.extra?.format !== undefined) {
       inferParams["format"] = params.extra.format;
       delete params.extra.format
     }
     // Spread any additional properties from params.extra if it exists and is not empty
     if (params.extra && Object.keys(params.extra).length > 0) {
       inferParams = { ...inferParams, ...params.extra };
     }
-    //console.log("Params", inferParams);
+    //console.log("INFER PARAMS", inferParams);
     let text = "";
     let data = {};
+    let stats: Record<string, any> = {};
     if (inferParams?.stream == true) {
       const body = JSON.stringify(inferParams);
       const buf = new Array<string>();
@@ -196,41 +205,57 @@ class OllamaProvider implements LmProvider {
         headers: { 'Content-Type': 'application/json' },
         body: body,
       });
-
       if (!response.body) {
         throw new Error("No response body")
       }
       const reader = response.body.getReader();
+      const decoder = new TextDecoder();
+      let lastBatch: Record<string, any> = {};
       while (true) {
         const { done, value } = await reader.read();
         if (done) break;
-        const raw = new TextDecoder().decode(value);
-        const d = JSON.parse(raw);
-        if (d["done"]) {
-          break
+        let raw = decoder.decode(value);
+        //console.log("RAW", raw);
+        const parts = raw.split('\n');
+        let pbuf = new Array();
+        for (const part of parts) {
+          try {
+            //console.log(part);
+            const p = JSON.parse(part);
+            lastBatch = p;
+            pbuf.push(p["response"]);
+          } catch (error) {
+            console.warn('invalid json: ', part)
+          }
         }
-        const t = d["response"];
+        const t = pbuf.join("");
         buf.push(t);
         if (this.onToken) {
           this.onToken(t);
         }
       }
-      text = buf.join("")
+      text = buf.join("");
+      stats = lastBatch;
     } else {
       const res = await this.api.post<Record<string, any>>("/api/generate", inferParams);
       if (res.ok) {
-        text = res.data.response
+        text = res.data.response;
+        stats = res.data;
       } else {
         throw new Error(`Error ${res.status} posting inference query ${res.data}`)
       }
     }
     if (parseJson) {
       data = parseJsonUtil(text, parseJsonFunc);
     }
+    delete stats.response;
+    delete stats.context;
+    delete stats.done;
+    //console.log("STATS", stats);
     const ir: InferenceResult = {
       text: text,
       data: data,
-      stats: {},
+      stats: stats,
     };
     return ir
   }

packages/evaluate/package.json

@@ -8,19 +8,19 @@
     "build": "rm -f dist/* && rollup -c"
   },
   "dependencies": {
-    "@locallm/api": "^0.0.21",
-    "modprompt": "^0.3.6"
+    "@locallm/api": "^0.0.29",
+    "modprompt": "^0.6.1"
   },
   "devDependencies": {
-    "@locallm/types": "^0.0.13",
+    "@locallm/types": "^0.0.14",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-terser": "^0.4.4",
-    "@rollup/plugin-typescript": "^11.1.5",
-    "@types/node": "^20.10.5",
-    "rollup": "^4.9.1",
+    "@rollup/plugin-typescript": "^11.1.6",
+    "@types/node": "^20.12.2",
+    "rollup": "^4.13.2",
     "ts-node": "^10.9.2",
     "tslib": "^2.6.2",
-    "typescript": "^5.3.3"
+    "typescript": "^5.4.3"
   },
   "type": "module",
   "files": [

packages/evaluate/src/evaluate.ts

@@ -1,7 +1,12 @@
-import { EvaluationFunction, EvaluationResult, TestResult } from "./interfaces.js";
-import { containsCodeBlock } from "./evaluators/code/main.js";
+import { InferenceParams } from "@locallm/types";
+import { EvaluationFunction, EvaluationResult, FinalEvaluationResult, TestResult } from "./interfaces.js";
+import { containsCodeBlock, isOnlyCodeBlock } from "./evaluators/code/main.js";
 import { containsValidJavascript, containsValidJson } from "./evaluators/code/js/main.js";
 import { containsText } from "./evaluators/text/containsText.js";
+import { isText } from "./evaluators/text/isText.js";
+import { startsWith } from "./evaluators/text/startsWith.js";
+import { maxLength } from "./evaluators/text/maxLength.js";
+import { containsOnlyOne } from "./evaluators/text/containsOnlyOne.js";
 
 class Evaluator {
   thresold = 100;
@@ -11,19 +16,27 @@ class Evaluator {
     this.thresold = thresold;
   }
 
-  run(response: string): TestResult {
+  run(name: string, prompt: string, response: string, inferParams: InferenceParams, stats: Record<string, any>): TestResult {
     const results: TestResult = {
+      name: name,
       pass: false,
       score: 0,
+      thresold: this.thresold,
+      prompt: prompt,
       output: response,
-      evaluations: new Array<EvaluationResult>(),
+      evaluations: new Array<FinalEvaluationResult>(),
+      inferParams: inferParams,
+      stats: stats,
     };
     this.evalFuncs.forEach((fp) => {
       const res = fp.func(response, fp.name, fp.param, fp.error);
       if (res.pass) {
         results.score += fp.passScore;
       }
-      results.evaluations.push(res);
+      results.evaluations.push({
+        ...res,
+        points: fp.passScore,
+      });
     });
     if (results.score >= this.thresold) {
       results.pass = true;
@@ -36,8 +49,33 @@ class Evaluator {
     return this
   }
 
+  isText(passScore: number, param: string | Array<string>, error: string | null = null): Evaluator {
+    this._stackEvalFunc(`Is text: ${param}`, passScore, isText, param, error);
+    return this
+  }
+
   containsText(passScore: number, param: string | Array<string>, error: string | null = null): Evaluator {
-    this._stackEvalFunc("Contains text", passScore, containsText, param, error);
+    this._stackEvalFunc(`Contains text ${param}`, passScore, containsText, param, error);
+    return this
+  }
+
+  containsOnlyOne(passScore: number, param: string | Array<string>, error: string | null = null): Evaluator {
+    this._stackEvalFunc(`Contains only one occurence of ${param}`, passScore, containsOnlyOne, param, error);
+    return this
+  }
+
+  startsWith(passScore: number, param: string | Array<string>, error: string | null = null): Evaluator {
+    this._stackEvalFunc(`Starts with ${param}`, passScore, startsWith, param, error);
+    return this
+  }
+
+  maxLength(passScore: number, param: number, error: string | null = null): Evaluator {
+    this._stackEvalFunc(`Max length ${param}`, passScore, maxLength, param, error);
+    return this
+  }
+
+  isOnlyCodeBlock(passScore: number, param: string | null = null, error: string | null = null): Evaluator {
+    this._stackEvalFunc("Is only a code block", passScore, isOnlyCodeBlock, param, error);
     return this
   }
 

packages/evaluate/src/evaluators/code/containsCodeBlock.ts

@@ -13,7 +13,7 @@ function containsCodeBlock(response: string, name: string, param: any, error: st
       if (error) {
         res.error = error
       } else {
-        res.error = "The output is not valid code"
+        res.error = "The output does not contain a code block"
       }
     } else {
       res.pass = true

packages/evaluate/src/evaluators/code/isOnlyCodeBlock.ts

@@ -0,0 +1,19 @@
+import { trimStr } from "../../utils.js";
+import { EvaluationResult } from "../../interfaces.js";
+
+function isOnlyCodeBlock(response: string, name: string, param: any, error: string | null = null): EvaluationResult {
+    const res: EvaluationResult = {
+        name: name,
+        pass: false,
+        error: null,
+    };
+    const trimedStr = trimStr(response);
+    if (trimedStr.startsWith("```") && trimedStr.endsWith("```")) {
+        res.pass = true;
+    } else {
+        res.error = "The output is not only a code block"
+    }
+    return res
+}
+
+export { isOnlyCodeBlock }

packages/evaluate/src/evaluators/code/js/containsValidJson.ts

@@ -10,6 +10,8 @@ function containsValidJson(response: string, name: string, param: any, error: st
   let code: string | null = null;
   if (response.includes("```")) {
     code = extractCodeBetweenTags(response);
+  } else {
+    code = response;
   }
   if (!code) {
     res.error = "The output does not contain a code block"

packages/evaluate/src/evaluators/code/main.ts

@@ -1,5 +1,7 @@
 import { containsCodeBlock } from "./containsCodeBlock.js";
+import { isOnlyCodeBlock } from "./isOnlyCodeBlock.js";
 
 export {
   containsCodeBlock,
+  isOnlyCodeBlock,
 }