feat(aio): switch llm for text

mudler · Mar 20, 2024 · 58324bc · 58324bc
1 parent 7d8f769
commit 58324bc
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 20 deletions.
diff --git a/aio/cpu/text-to-text.yaml b/aio/cpu/text-to-text.yaml
@@ -1,4 +1,4 @@
-name: phi-2
+name: gpt-3.5-turbo
 context_size: 2048
 f16: true
 gpu_layers: 90
@@ -7,14 +7,7 @@ trimsuffix:
 - "\n"
 parameters:
   model: huggingface://TheBloke/phi-2-GGUF/phi-2.Q8_0.gguf
-  temperature: 0.2
-  top_k: 40
-  top_p: 0.95
-  seed: -1
-
-mirostat: 2
-mirostat_eta: 1.0
-mirostat_tau: 1.0
+
 template:
   chat: &template |-
     Instruct: {{.Input}}

diff --git a/aio/gpu-8g/text-to-text.yaml b/aio/gpu-8g/text-to-text.yaml
@@ -1,20 +1,39 @@
-name: mistral-openorca
+name: gpt-3.5-turbo
 mmap: true
 parameters:
-  model: huggingface://TheBloke/Mistral-7B-OpenOrca-GGUF/mistral-7b-openorca.Q6_K.gguf
-  temperature: 0.2
-  top_k: 40
-  top_p: 0.95
-  seed: -1
-mirostat: 2
-mirostat_eta: 1.0
-mirostat_tau: 1.0
+  model: huggingface://NousResearch/Hermes-2-Pro-Mistral-7B-GGUF/Hermes-2-Pro-Mistral-7B.Q6_K.gguf
 
+roles:
+  assistant_function_call: assistant
+  function: tool
 template:
   chat_message: |
-    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
+    <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "function"}}{{.Role}}{{else if eq .RoleName "user"}}user{{end}}
+    {{ if eq .RoleName "assistant_function_call" }}<tool_call>{{end}}
+    {{ if eq .RoleName "function" }}<tool_result>{{end}}
     {{if .Content}}{{.Content}}{{end}}
+    {{if .FunctionCall}}{{toJson .FunctionCall}}{{end}}
+    {{ if eq .RoleName "assistant_function_call" }}</tool_call>{{end}}
+    {{ if eq .RoleName "function" }}</tool_result>{{end}}
     <|im_end|>
+  # https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B-GGUF#prompt-format-for-function-calling
+  function: |
+    <|im_start|>system
+    You are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: 
+    <tools>
+    {{range .Functions}}
+    {'type': 'function', 'function': {'name': '{{.Name}}', 'description': '{{.Description}}', 'parameters': {{toJson .Parameters}} }}
+    {{end}}
+    </tools> 
+    Use the following pydantic model json schema for each tool call you will make: 
+    {'title': 'FunctionCall', 'type': 'object', 'properties': {'arguments': {'title': 'Arguments', 'type': 'object'}, 'name': {'title': 'Name', 'type': 'string'}}, 'required': ['arguments', 'name']} 
+    For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
+    <tool_call>
+    {'arguments': <args-dict>, 'name': <function-name>}
+    </tool_call><|im_end|>
+    {{.Input}}
+    <|im_start|>assistant
+    <tool_call>
   chat: |
     {{.Input}}
     <|im_start|>assistant
@@ -27,6 +46,6 @@ stopwords:
 - <dummy32000>
 usage: |
       curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
-          "model": "mistral-openorca",
+          "model": "hermes-2-pro-mistral",
           "messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}]
       }'