From b0de1437c0ef7c5a538079a1d322bc816a6e5bef Mon Sep 17 00:00:00 2001 From: "zhouxiao.shaw" Date: Thu, 29 Aug 2024 11:01:07 +0800 Subject: [PATCH 1/4] fix(ai-plan): optimize KeyboardPress plan type reduce hallucinations --- packages/midscene/src/ai-model/automation/planning.ts | 6 ++++-- .../web-integration/tests/ai/e2e/ai-online-order.spec.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/midscene/src/ai-model/automation/planning.ts b/packages/midscene/src/ai-model/automation/planning.ts index 64e9d457..64b7d042 100644 --- a/packages/midscene/src/ai-model/automation/planning.ts +++ b/packages/midscene/src/ai-model/automation/planning.ts @@ -12,8 +12,10 @@ export function systemPromptToTaskPlanning() { * param: null * type: 'Hover', hover the previous element found * param: null - * type: 'Input', 'KeyboardPress', input something or press a key - * param: { value: string }, Based on the value of the existing input, give the required value of the final input (may add or subtract content from the original),Typing in the input box usually requires clicking input first. the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key. + * type: 'Input', input something + * param: { value: string }, Based on the value of the existing input, give the required value of the final input (may add or subtract content from the original),Typing in the input box usually requires clicking input first. + * type: 'KeyboardPress', press a key + * param: { value: string }, the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key. * type: 'Scroll' * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' } * type: 'Error' diff --git a/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts b/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts index c00b2331..c83a60e0 100644 --- a/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts +++ b/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts @@ -15,9 +15,9 @@ test('ai online order', async ({ ai, page, aiQuery }) => { await ai('点击左上角语言切换按钮(英文、中文)，在弹出的下拉列表中点击中文'); await ai('向下滚动一屏'); await ai('直接点击多肉葡萄的规格按钮'); - await ai('点击不使用吸管、点击冰沙推荐、点击正常冰推荐'); + await ai('点击每个选项的勾选框，如果有推荐的优先点推荐'); await ai('向下滚动一屏'); - await ai('点击标准甜、点击绿妍（推荐）、点击标准口味'); + await ai('点击每个选项的勾选框，如果有推荐的优先点推荐'); await ai('滚动到最下面'); await ai('点击选好了按钮'); await ai('点击右上角商品图标按钮'); From b0a499bc734556491233b8c58acdbc2ad647e31e Mon Sep 17 00:00:00 2001 From: "zhouxiao.shaw" Date: Thu, 29 Aug 2024 11:59:34 +0800 Subject: [PATCH 2/4] chore: update test case --- .../tests/ai/e2e/ai-auto-todo.spec.ts | 4 +--- .../tests/ai/e2e/ai-online-order.spec.ts | 6 +++--- .../ai-report-1-report.txt | 13 +++---------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts b/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts index 0a8f7fdf..15ff820b 100644 --- a/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts +++ b/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts @@ -8,7 +8,7 @@ test.beforeEach(async ({ page }) => { const CACHE_TIME_OUT = Boolean(process.env.MIDSCENE_CACHE) && Boolean(process.env.GITHUB_ACTIONS); -test('ai todo', async ({ ai, aiQuery, aiWaitFor }) => { +test('ai todo', async ({ ai, aiQuery }) => { if (CACHE_TIME_OUT) { test.setTimeout(1000 * 30); } @@ -18,8 +18,6 @@ test('ai todo', async ({ ai, aiQuery, aiWaitFor }) => { 'Add "JS today" to the existing content of the task box and press enter', ); - await aiWaitFor('the input box for task title is empty now'); - await ai( 'Enter "Learn Rust tomorrow" in the task box, then press Enter to create', ); diff --git a/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts b/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts index c83a60e0..fb572e5d 100644 --- a/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts +++ b/packages/web-integration/tests/ai/e2e/ai-online-order.spec.ts @@ -15,11 +15,11 @@ test('ai online order', async ({ ai, page, aiQuery }) => { await ai('点击左上角语言切换按钮(英文、中文)，在弹出的下拉列表中点击中文'); await ai('向下滚动一屏'); await ai('直接点击多肉葡萄的规格按钮'); - await ai('点击每个选项的勾选框，如果有推荐的优先点推荐'); + await ai('点击不使用吸管、点击冰沙推荐、点击正常冰推荐'); await ai('向下滚动一屏'); - await ai('点击每个选项的勾选框，如果有推荐的优先点推荐'); + await ai('点击标准甜、点击绿妍（推荐）、点击标准口味'); await ai('滚动到最下面'); - await ai('点击选好了按钮'); + await ai('点击页面下边的“选好了”按钮'); await ai('点击右上角商品图标按钮'); const cardDetail = await aiQuery({ diff --git a/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt b/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt index 24a73b5e..7dbdda2e 100644 --- a/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt +++ b/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt @@ -18,20 +18,13 @@ "Action / KeyboardPress" ] }, - { - "title": "waitFor: the input box for task title is empty now", - "actions": [ - "Insight / Assert", - "Insight / Assert", - "Action / Sleep", - "Insight / Assert" - ] - }, { "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create", "actions": [ "Planning", - "Insight / Locate" + "Insight / Locate", + "Action / Input", + "Action / KeyboardPress" ] } ] \ No newline at end of file From 52b3ead56a88aadeebc404bdf23e406b11e9f912 Mon Sep 17 00:00:00 2001 From: "zhouxiao.shaw" Date: Thu, 29 Aug 2024 12:09:07 +0800 Subject: [PATCH 3/4] chore: update test case --- .../todo-report.spec.ts-snapshots/ai-report-1-report.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt b/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt index 7dbdda2e..eb27bd34 100644 --- a/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt +++ b/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt @@ -18,13 +18,4 @@ "Action / KeyboardPress" ] }, - { - "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create", - "actions": [ - "Planning", - "Insight / Locate", - "Action / Input", - "Action / KeyboardPress" - ] - } ] \ No newline at end of file From 65d51df7cb3d8f18a86e04f8938318d1f348e996 Mon Sep 17 00:00:00 2001 From: "zhouxiao.shaw" Date: Thu, 29 Aug 2024 14:31:16 +0800 Subject: [PATCH 4/4] chore: update test case --- .../ai-report-1-report.txt | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt b/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt index eb27bd34..a3b5432c 100644 --- a/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt +++ b/packages/web-integration/tests/ai/report/todo-report.spec.ts-snapshots/ai-report-1-report.txt @@ -18,4 +18,22 @@ "Action / KeyboardPress" ] }, -] \ No newline at end of file + { + "title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create", + "actions": [ + "Planning", + "Insight / Locate", + "Action / Input", + "Action / KeyboardPress" + ] + }, + { + "title": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create", + "actions": [ + "Planning", + "Insight / Locate", + "Action / Input", + "Action / KeyboardPress" + ] + } +]