fix(ai-plan): optimize KeyboardPress plan type reduce hallucinations (#…

…79)
web-infra-dev · Aug 29, 2024 · 3ad8e9c · 3ad8e9c
1 parent 84ee713
commit 3ad8e9c
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 14 deletions.
diff --git a/packages/midscene/src/ai-model/automation/planning.ts b/packages/midscene/src/ai-model/automation/planning.ts
@@ -12,8 +12,10 @@ export function systemPromptToTaskPlanning() {
     * param: null
   * type: 'Hover', hover the previous element found
     * param: null
-  * type: 'Input', 'KeyboardPress', input something or press a key
-    * param: { value: string }, Based on the value of the existing input, give the required value of the final input (may add or subtract content from the original),Typing in the input box usually requires clicking input first. the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
+  * type: 'Input', input something
+    * param: { value: string }, Based on the value of the existing input, give the required value of the final input (may add or subtract content from the original),Typing in the input box usually requires clicking input first.
+  * type: 'KeyboardPress',  press a key
+    * param: { value: string },  the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
   * type: 'Scroll'
     * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' }
   * type: 'Error'

diff --git a/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts b/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts
@@ -8,7 +8,7 @@ test.beforeEach(async ({ page }) => {
 const CACHE_TIME_OUT =
   Boolean(process.env.MIDSCENE_CACHE) && Boolean(process.env.GITHUB_ACTIONS);
 
-test('ai todo', async ({ ai, aiQuery, aiWaitFor }) => {
+test('ai todo', async ({ ai, aiQuery }) => {
   if (CACHE_TIME_OUT) {
     test.setTimeout(1000 * 30);
   }
@@ -18,8 +18,6 @@ test('ai todo', async ({ ai, aiQuery, aiWaitFor }) => {
     'Add "JS today" to the existing content of the task box and press enter',
   );
 
-  await aiWaitFor('the input box for task title is empty now');
-
   await ai(
     'Enter "Learn Rust tomorrow" in the task box, then press Enter to create',
   );

diff --git a/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts b/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts
@@ -19,7 +19,7 @@ test('ai online order', async ({ ai, page, aiQuery }) => {
   await ai('向下滚动一屏');
   await ai('点击标准甜、点击绿妍（推荐）、点击标准口味');
   await ai('滚动到最下面');
-  await ai('点击选好了按钮');
+  await ai('点击页面下边的“选好了”按钮');
   await ai('点击右上角商品图标按钮');
 
   const cardDetail = await aiQuery({

diff --git a/...ages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt b/...ages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt
@@ -19,19 +19,21 @@
         ]
     },
     {
-        "title": "waitFor: the input box for task title is empty now",
+        "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create",
         "actions": [
-            "Insight / Assert",
-            "Insight / Assert",
-            "Action / Sleep",
-            "Insight / Assert"
+            "Planning",
+            "Insight / Locate",
+            "Action / Input",
+            "Action / KeyboardPress"
         ]
     },
     {
-        "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create",
+        "title": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create",
         "actions": [
             "Planning",
-            "Insight / Locate"
+            "Insight / Locate",
+            "Action / Input",
+            "Action / KeyboardPress"
         ]
     }
-]
+]