From 3c2b18675d88db842ae9abc11f02d9b84c60d2ce Mon Sep 17 00:00:00 2001 From: yutao Date: Tue, 20 Aug 2024 11:56:55 +0800 Subject: [PATCH 01/26] feat: add --- packages/midscene/src/types.ts | 20 +++- packages/web-integration/src/common/agent.ts | 22 ++++- packages/web-integration/src/common/tasks.ts | 99 +++++++++++++++++-- .../tests/ai/puppeteer/showcase.test.ts | 6 +- 4 files changed, 134 insertions(+), 13 deletions(-) diff --git a/packages/midscene/src/types.ts b/packages/midscene/src/types.ts index 51975432..750f56f8 100644 --- a/packages/midscene/src/types.ts +++ b/packages/midscene/src/types.ts @@ -166,6 +166,15 @@ export type ElementById = (id: string) => BaseElement | null; export type InsightAssertionResponse = AIAssertionResponse; +/** + * agent + */ + +export interface AgentWaitForOpt { + checkIntervalMs?: number; + timeoutMs?: number; +} + /** * planning * @@ -182,7 +191,9 @@ export interface PlanningAction { | 'Scroll' | 'Error' | 'Assert' - | 'Sleep'; + | 'AssertWithoutThrow' + | 'Sleep' + | 'WaitFor'; param: ParamType; } @@ -213,6 +224,13 @@ export interface PlanningActionParamSleep { timeMs: number; } +export interface PlanningActionParamError { + thought: string; +} + +export type PlanningActionParamWaitFor = AgentWaitForOpt & { + assertion: string; +}; /** * misc */ diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts index 35545e57..c003ec47 100644 --- a/packages/web-integration/src/common/agent.ts +++ b/packages/web-integration/src/common/agent.ts @@ -1,11 +1,14 @@ import type { WebPage } from '@/common/page'; -import type { ExecutionDump, GroupedActionDump } from '@midscene/core'; +import type { + AgentWaitForOpt, + ExecutionDump, + GroupedActionDump, +} from '@midscene/core'; import { groupedActionDumpFileExt, stringifyDumpData, writeLogFile, } from '@midscene/core/utils'; -import dayjs from 'dayjs'; import { PageTaskExecutor } from '../common/tasks'; import type { AiTaskCache } from './task-cache'; import { printReportMsg, reportFileName } from './utils'; @@ -115,6 +118,21 @@ export class PageAgent { } } + async aiWaitFor(assertion: string, opt?: AgentWaitForOpt) { + const { executor } = await this.taskExecutor.waitFor(assertion, { + timeoutMs: opt?.timeoutMs || 30 * 1000, + checkIntervalMs: opt?.checkIntervalMs || 3 * 1000, + assertion, + }); + this.appendExecutionDump(executor.dump()); + this.writeOutActionDumps(); + + if (executor.isInErrorState()) { + const errorTask = executor.latestErrorTask(); + throw new Error(`${errorTask?.error}\n${errorTask?.errorStack}`); + } + } + async ai(taskPrompt: string, type = 'action') { if (type === 'action') { return this.aiAction(taskPrompt); diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index 13148816..6bc936db 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -10,10 +10,10 @@ import Insight, { type ExecutionTaskInsightQueryApply, type ExecutionTaskPlanningApply, Executor, + plan, type InsightAssertionResponse, type InsightDump, type InsightExtractParam, - plan, type PlanningAction, type PlanningActionParamAssert, type PlanningActionParamHover, @@ -21,10 +21,11 @@ import Insight, { type PlanningActionParamScroll, type PlanningActionParamSleep, type PlanningActionParamTap, + type PlanningActionParamWaitFor, + type PlanningActionParamError, } from '@midscene/core'; import { base64Encoded } from '@midscene/core/image'; import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils'; -import type { ChatCompletionMessageParam } from 'openai/resources'; import type { KeyInput, Page as PuppeteerPage } from 'puppeteer'; import type { WebElementInfo } from '../web-element'; import { type AiTaskCache, TaskCache } from './task-cache'; @@ -150,7 +151,7 @@ export class PageTaskExecutor { }; return taskFind; } - if (plan.type === 'Assert') { + if (plan.type === 'Assert' || plan.type === 'AssertWithoutThrow') { const assertPlan = plan as PlanningAction; const taskAssert: ExecutionTaskApply = { type: 'Insight', @@ -167,7 +168,7 @@ export class PageTaskExecutor { assertPlan.param.assertion, ); - if (!assertion.pass) { + if (!assertion.pass && plan.type === 'Assert') { task.output = assertion; task.log = { dump: insightDump, @@ -299,8 +300,22 @@ export class PageTaskExecutor { return taskActionSleep; } if (plan.type === 'Error') { - throw new Error(`Got a task plan with type Error: ${plan.thought}`); + const taskActionError: ExecutionTaskActionApply = + { + type: 'Action', + subType: 'Error', + param: plan.param, + executor: async (taskParam) => { + assert( + taskParam.thought, + 'An error occurred, but no thought provided', + ); + throw new Error(taskParam.thought); + }, + }; + return taskActionError; } + throw new Error(`Unknown or Unsupported task type: ${plan.type}`); }) .map((task: ExecutionTaskApply) => { @@ -314,7 +329,6 @@ export class PageTaskExecutor { userPrompt: string /* , actionInfo?: { actionType?: EventActions[number]['action'] } */, ): Promise { const taskExecutor = new Executor(userPrompt); - taskExecutor.description = userPrompt; let plans: PlanningAction[] = []; const planningTask: ExecutionTaskPlanningApply = { @@ -386,7 +400,6 @@ export class PageTaskExecutor { const description = typeof demand === 'string' ? demand : JSON.stringify(demand); const taskExecutor = new Executor(description); - taskExecutor.description = description; const queryTask: ExecutionTaskInsightQueryApply = { type: 'Insight', subType: 'Query', @@ -418,9 +431,8 @@ export class PageTaskExecutor { async assert( assertion: string, ): Promise> { - const description = assertion; + const description = `assert: ${assertion}`; const taskExecutor = new Executor(description); - taskExecutor.description = description; const assertionPlan: PlanningAction = { type: 'Assert', param: { @@ -437,4 +449,73 @@ export class PageTaskExecutor { executor: taskExecutor, }; } + + async waitFor( + assertion: string, + opt: PlanningActionParamWaitFor, + ): Promise> { + const description = `waitFor: ${assertion}`; + const taskExecutor = new Executor(description); + const { timeoutMs, checkIntervalMs } = opt; + + assert(assertion, 'No assertion for waitFor'); + assert(timeoutMs, 'No timeoutMs for waitFor'); + assert(checkIntervalMs, 'No checkIntervalMs for waitFor'); + + const overallStartTime = Date.now(); + let startTime = Date.now(); + let errorThought = ''; + while (Date.now() - overallStartTime < timeoutMs) { + startTime = Date.now(); + const assertPlan: PlanningAction = { + type: 'AssertWithoutThrow', + param: { + assertion, + }, + }; + const assertTask = await this.convertPlanToExecutable([assertPlan]); + await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0])); + const output: InsightAssertionResponse = await taskExecutor.flush(); + console.log(output); + + if (output.pass) { + return { + output: undefined, + executor: taskExecutor, + }; + } + + errorThought = output.thought; + const now = Date.now(); + if (now - startTime < checkIntervalMs) { + const timeRemaining = checkIntervalMs - (now - startTime); + const sleepPlan: PlanningAction = { + type: 'Sleep', + param: { + timeMs: timeRemaining, + }, + }; + const sleepTask = await this.convertPlanToExecutable([sleepPlan]); + await taskExecutor.append( + this.wrapExecutorWithScreenshot(sleepTask[0]), + ); + await taskExecutor.flush(); + } + } + + // throw an error using taskExecutor + const errorPlan: PlanningAction = { + type: 'Error', + param: { + thought: `waitFor timeout: ${errorThought}`, + }, + }; + const errorTask = await this.convertPlanToExecutable([errorPlan]); + await taskExecutor.append(errorTask[0]); + await taskExecutor.flush(); + return { + output: undefined, + executor: taskExecutor, + }; + } } diff --git a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts b/packages/web-integration/tests/ai/puppeteer/showcase.test.ts index 915db144..fbfb0c31 100644 --- a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts +++ b/packages/web-integration/tests/ai/puppeteer/showcase.test.ts @@ -19,7 +19,11 @@ describe( 'type "standard_user" in user name input, type "secret_sauce" in password, click "Login"', ); - await sleep(2000); + await expect(async () => { + await mid.aiWaitFor('there is a cookie prompt in the UI', { + timeoutMs: 10 * 1000, + }); + }).rejects.toThrowError(); // find the items const items = await mid.aiQuery( From 8b1880280f6278cfa1654a5cb85c20a291dd6e09 Mon Sep 17 00:00:00 2001 From: yutao Date: Tue, 20 Aug 2024 14:01:03 +0800 Subject: [PATCH 02/26] feat: add --- packages/midscene/src/action/executor.ts | 3 +-- packages/midscene/src/types.ts | 2 +- .../midscene/tests/ai/executor/index.test.ts | 4 ++-- packages/visualizer/src/component/misc.tsx | 10 ++++++++- packages/visualizer/src/component/sidebar.tsx | 7 ++++++- packages/visualizer/src/utils.ts | 2 +- packages/web-integration/src/common/tasks.ts | 21 +++++++++++-------- .../tests/ai/puppeteer/showcase.test.ts | 2 +- 8 files changed, 33 insertions(+), 18 deletions(-) diff --git a/packages/midscene/src/action/executor.ts b/packages/midscene/src/action/executor.ts index 5300287f..a8510b41 100644 --- a/packages/midscene/src/action/executor.ts +++ b/packages/midscene/src/action/executor.ts @@ -124,8 +124,7 @@ export class Executor { } Object.assign(task, returnValue); - - task.status = 'success'; + task.status = 'finished'; task.timing.end = Date.now(); task.timing.cost = task.timing.end - task.timing.start; taskIndex++; diff --git a/packages/midscene/src/types.ts b/packages/midscene/src/types.ts index 750f56f8..db61cef6 100644 --- a/packages/midscene/src/types.ts +++ b/packages/midscene/src/types.ts @@ -311,7 +311,7 @@ export type ExecutionTask< ? TaskLog : unknown > & { - status: 'pending' | 'running' | 'success' | 'failed' | 'cancelled'; + status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled'; error?: string; errorStack?: string; timing?: { diff --git a/packages/midscene/tests/ai/executor/index.test.ts b/packages/midscene/tests/ai/executor/index.test.ts index 4338a6c9..8db3f0fd 100644 --- a/packages/midscene/tests/ai/executor/index.test.ts +++ b/packages/midscene/tests/ai/executor/index.test.ts @@ -87,7 +87,7 @@ describe('executor', () => { expect(element).toBeTruthy(); expect(tasks.length).toBe(inputTasks.length); - expect(tasks[0].status).toBe('success'); + expect(tasks[0].status).toBe('finished'); expect(tasks[0].output).toMatchSnapshot(); expect(tasks[0].log?.dump).toBeTruthy(); expect(tasks[0].timing?.end).toBeTruthy(); @@ -151,7 +151,7 @@ describe('executor', () => { expect(initExecutor.status).toBe('completed'); expect(initExecutor.tasks.length).toBe(3); - expect(initExecutor.tasks[2].status).toBe('success'); + expect(initExecutor.tasks[2].status).toBe('finished'); // append while completed initExecutor.append(actionTask); diff --git a/packages/visualizer/src/component/misc.tsx b/packages/visualizer/src/component/misc.tsx index e15000ba..4282839a 100644 --- a/packages/visualizer/src/component/misc.tsx +++ b/packages/visualizer/src/component/misc.tsx @@ -5,6 +5,7 @@ import { CloseCircleFilled, LogoutOutlined, MinusOutlined, + WarningFilled, } from '@ant-design/icons'; export function timeCostStrElement(timeCost?: number) { @@ -32,13 +33,20 @@ export function timeCostStrElement(timeCost?: number) { export const iconForStatus = (status: string): JSX.Element => { switch (status) { - case 'success': + case 'finished': case 'passed': return ( ); + + case 'finishedWithWarning': + return ( + + + + ); case 'failed': case 'timedOut': case 'interrupted': diff --git a/packages/visualizer/src/component/sidebar.tsx b/packages/visualizer/src/component/sidebar.tsx index ab345627..e4ff463b 100644 --- a/packages/visualizer/src/component/sidebar.tsx +++ b/packages/visualizer/src/component/sidebar.tsx @@ -23,6 +23,11 @@ const SideItem = (props: { statusText = timeCostStrElement(task.timing.cost); } + const statusIcon = + task.status === 'finished' && task.error + ? iconForStatus('finishedWithWarning') + : iconForStatus(task.status); + return (
{' '}
- {iconForStatus(task.status)} + {statusIcon}
{typeStr(task)}
{statusText}
diff --git a/packages/visualizer/src/utils.ts b/packages/visualizer/src/utils.ts index 11b333ab..1068d825 100644 --- a/packages/visualizer/src/utils.ts +++ b/packages/visualizer/src/utils.ts @@ -16,7 +16,7 @@ export function insightDumpToExecutionDump( const task: ExecutionTaskInsightLocate = { type: 'Insight', subType: insightDump.type === 'locate' ? 'Locate' : 'Query', - status: insightDump.error ? 'failed' : 'success', + status: insightDump.error ? 'failed' : 'finished', param: { ...(insightDump.userQuery.element ? { query: insightDump.userQuery } diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index 6bc936db..bed13d1b 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -168,14 +168,18 @@ export class PageTaskExecutor { assertPlan.param.assertion, ); - if (!assertion.pass && plan.type === 'Assert') { - task.output = assertion; - task.log = { - dump: insightDump, - }; - throw new Error( - assertion.thought || 'Assertion failed without reason', - ); + if (!assertion.pass) { + if (plan.type === 'Assert') { + task.output = assertion; + task.log = { + dump: insightDump, + }; + throw new Error( + assertion.thought || 'Assertion failed without reason', + ); + } + + task.error = assertion.thought; } return { @@ -476,7 +480,6 @@ export class PageTaskExecutor { const assertTask = await this.convertPlanToExecutable([assertPlan]); await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0])); const output: InsightAssertionResponse = await taskExecutor.flush(); - console.log(output); if (output.pass) { return { diff --git a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts b/packages/web-integration/tests/ai/puppeteer/showcase.test.ts index fbfb0c31..c94e3611 100644 --- a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts +++ b/packages/web-integration/tests/ai/puppeteer/showcase.test.ts @@ -72,8 +72,8 @@ describe( ); console.log('Github service status', result); + // obviously there is no food delivery service on Github expect(async () => { - // // there is no food delivery service on Github await mid.aiAssert( 'there is a "food delivery" service on page and is in normal state', ); From acb8cb5d9be4bf2acf2135280848f26a8a9553fc Mon Sep 17 00:00:00 2001 From: yutao Date: Tue, 20 Aug 2024 14:08:39 +0800 Subject: [PATCH 03/26] feat: add for playwright --- .../web-integration/src/playwright/index.ts | 13 +++ .../tests/ai/e2e/ai-auto-todo.spec.ts | 4 +- .../tests/ai/e2e/todo-app-midscene.spec.ts | 98 ------------------- 3 files changed, 16 insertions(+), 99 deletions(-) delete mode 100644 packages/web-integration/tests/ai/e2e/todo-app-midscene.spec.ts diff --git a/packages/web-integration/src/playwright/index.ts b/packages/web-integration/src/playwright/index.ts index 980b9406..1dbe9787 100644 --- a/packages/web-integration/src/playwright/index.ts +++ b/packages/web-integration/src/playwright/index.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto'; import { PageAgent } from '@/common/agent'; import type { WebPage } from '@/common/page'; +import type { AgentWaitForOpt } from '@midscene/core/.'; import type { TestInfo, TestType } from '@playwright/test'; import type { Page as PlaywrightPage } from 'playwright'; import type { PageTaskExecutor } from '../common/tasks'; @@ -128,6 +129,17 @@ export const PlaywrightAiFixture = () => { }); updateDumpAnnotation(testInfo, agent.dumpDataString()); }, + aiWaitFor: async ( + { page }: { page: PlaywrightPage }, + use: any, + testInfo: TestInfo, + ) => { + const agent = agentForPage(page, testInfo); + await use(async (assertion: string, opt?: AgentWaitForOpt) => { + await agent.aiWaitFor(assertion, opt); + }); + updateDumpAnnotation(testInfo, agent.dumpDataString()); + }, }; }; @@ -139,4 +151,5 @@ export type PlayWrightAiFixtureType = { aiAction: (taskPrompt: string) => ReturnType; aiQuery: (demand: any) => Promise; aiAssert: (assertion: string, errorMsg?: string) => Promise; + aiWaitFor: (assertion: string, opt?: AgentWaitForOpt) => Promise; }; diff --git a/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts b/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts index dab52637..56b097d5 100644 --- a/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts +++ b/packages/web-integration/tests/ai/e2e/ai-auto-todo.spec.ts @@ -5,11 +5,13 @@ test.beforeEach(async ({ page }) => { await page.goto('https://todomvc.com/examples/react/dist/'); }); -test('ai todo', async ({ ai, aiQuery }) => { +test('ai todo', async ({ ai, aiQuery, aiWaitFor }) => { await ai( 'Enter "Learn JS today" in the task box, then press Enter to create', ); + await aiWaitFor('the input box for task title is empty now'); + await ai( 'Enter "Learn Rust tomorrow" in the task box, then press Enter to create', ); diff --git a/packages/web-integration/tests/ai/e2e/todo-app-midscene.spec.ts b/packages/web-integration/tests/ai/e2e/todo-app-midscene.spec.ts deleted file mode 100644 index 4a966bac..00000000 --- a/packages/web-integration/tests/ai/e2e/todo-app-midscene.spec.ts +++ /dev/null @@ -1,98 +0,0 @@ -// import { test, expect, type Page } from '@playwright/test'; -// import Insight, { TextElement, query } from 'midscene'; -// import { retrieveElements, retrieveOneElement } from 'midscene/query'; - -// test.beforeEach(async ({ page }) => { -// await page.goto('https://todomvc.com/examples/react/dist/'); -// }); - -// const TODO_ITEMS = ['buy some cheese', 'feed the cat', 'book a doctors appointment']; - -// interface InputBoxSection { -// element: TextElement; -// toggleAllBtn: TextElement; -// placeholder: string; -// inputValue: string; -// } - -// interface TodoItem { -// name: string; -// finished: boolean; -// } - -// interface ControlLayerSection { -// numbersLeft: number; -// tipElement: TextElement; -// controlElements: TextElement[]; -// } - -// // A comprehensive parser for page content -// const parsePage = async (page: Page) => { -// const insight = await Insight.fromPlaywrightPage(page); -// const todoListPage = await insight.segment({ -// 'input-box': query('an input box to type item and a "toggle-all" button', { -// element: retrieveOneElement('input box'), -// toggleAllBtn: retrieveOneElement('toggle all button, if exists'), -// placeholder: 'placeholder string in the input box, string, if exists', -// inputValue: 'the value in the input box, string, if exists', -// }), -// 'todo-list': query<{ todoItems: TodoItem[] }>('a list with todo-data (if exists)', { -// todoItems: '{name: string, finished: boolean}[]', -// }), -// 'control-layer': query('status and control layer of todo (if exists)', { -// numbersLeft: 'number', -// tipElement: retrieveOneElement( -// 'the element indicates the number of remaining items, like ` items left`', -// ), -// controlElements: retrieveElements('control elements, used to filter items'), -// }), -// }); - -// return todoListPage; -// }; - -// test.describe('New Todo', () => { -// test('should allow me to add todo items', async ({ page }) => { -// // add a todo item -// const todoPage = await parsePage(page); -// const inputBox = todoPage['input-box']; -// expect(inputBox).toBeTruthy(); - -// await page.mouse.click(...inputBox!.element.center); -// await page.keyboard.type(TODO_ITEMS[0], { delay: 100 }); -// await page.keyboard.press('Enter'); - -// // update page parsing result, and check the interface -// const todoPage2 = await parsePage(page); -// expect(todoPage2['input-box'].inputValue).toBeFalsy(); -// expect(todoPage2['input-box'].placeholder).toBeTruthy(); -// expect(todoPage2['todo-list'].todoItems.length).toBe(1); -// expect(todoPage2['todo-list'].todoItems[0].name).toBe(TODO_ITEMS[0]); - -// // add another item -// await page.mouse.click(...todoPage2['input-box'].element.center); -// await page.keyboard.type(TODO_ITEMS[1], { delay: 100 }); -// await page.keyboard.press('Enter'); - -// // update page parsing result -// const todoPage3 = await parsePage(page); -// const items = todoPage3['todo-list'].todoItems; -// expect(items.length).toBe(2); -// expect(items[1].name).toEqual(TODO_ITEMS[1]); -// expect(items.some((item) => item.finished)).toBeFalsy(); -// expect(todoPage3['control-layer'].numbersLeft).toBe(2); - -// // will mark all as completed -// const toggleBtn = todoPage3['input-box'].toggleAllBtn; -// expect(toggleBtn).toBeTruthy(); -// expect(todoPage3['todo-list'].todoItems.filter((item) => item.finished).length).toBe(0); - -// await page.mouse.click(...toggleBtn!.center, { delay: 500 }); -// await page.waitForTimeout(3000); - -// const todoPage4 = await parsePage(page); -// const allItems = todoPage4['todo-list'].todoItems; -// expect(allItems.length).toBe(2); -// expect(allItems.filter((item) => item.finished).length).toBe(allItems.length); -// }); -// }); From eb8c7321c332111294d31258ef6b1c3f907ef3f3 Mon Sep 17 00:00:00 2001 From: yutao Date: Tue, 20 Aug 2024 14:28:15 +0800 Subject: [PATCH 04/26] feat: add docs for 'aiWaitFor' --- apps/site/docs/en/docs/usage/API.md | 6 ++++++ apps/site/docs/zh/docs/usage/API.md | 7 ++++++- packages/midscene/src/types.ts | 3 +-- packages/web-integration/src/common/agent.ts | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/site/docs/en/docs/usage/API.md b/apps/site/docs/en/docs/usage/API.md index a7f5b7f2..6430cf1b 100644 --- a/apps/site/docs/en/docs/usage/API.md +++ b/apps/site/docs/en/docs/usage/API.md @@ -100,6 +100,12 @@ expect(onesieItem.price).toBe(7.99); ``` ::: +### `.aiWaitFor(assertion: string, {timeoutMs?: number, checkIntervalMs?: number })` - wait until the assertion is met + +`.aiWaitFor` will help you check if your assertion has been met or an timeout error occurred. Considering the AI service cost, the check interval will not exceed `checkIntervalMs` milliseconds. The default config sets `timeoutMs` to 15 seconds and `checkIntervalMs` to 3 seconds: i.e. check at most 5 times if all assertions fail and the AI service always responds immediately. + +When considering the time required for the AI service, `.aiWaitFor` may not be very efficient. Using a simple `sleep` method might be a useful alternative to `waitFor`. + ## Use LangSmith (Optional) LangSmith is a platform designed to debug the LLMs. To integrate LangSmith, please follow these steps: diff --git a/apps/site/docs/zh/docs/usage/API.md b/apps/site/docs/zh/docs/usage/API.md index 575765a0..b158a406 100644 --- a/apps/site/docs/zh/docs/usage/API.md +++ b/apps/site/docs/zh/docs/usage/API.md @@ -98,11 +98,16 @@ expect(onesieItem.price).toBe(7.99); ``` ::: +### `.aiWaitFor(assertion: string, {timeoutMs?: number, checkIntervalMs?: number })` - 等待断言执行成功 + +`.aiWaitFor` 帮助你检查你的断言是否满足，或是是否发生了超时错误。考虑到 AI 服务的成本，检查间隔不会超过 `checkIntervalMs` 毫秒。默认配置将 `timeoutMs` 设为 15 秒，`checkIntervalMs` 设为 3 秒：也就是说，如果所有断言都失败，并且 AI 服务总是立即响应，则最多检查 5 次。 + +考虑到 AI 服务的时间消耗，`.aiWaitFor` 并不是一个特别高效的方法。使用一个普通的 `sleep` 可能是替代 `waitFor` 的另一种方式。 + ## 使用 LangSmith （可选） LangSmith 是一个用于调试大语言模型的平台。想要集成 LangSmith，请按以下步骤操作： - ```bash # 设置环境变量 diff --git a/packages/midscene/src/types.ts b/packages/midscene/src/types.ts index db61cef6..3330e45f 100644 --- a/packages/midscene/src/types.ts +++ b/packages/midscene/src/types.ts @@ -192,8 +192,7 @@ export interface PlanningAction { | 'Error' | 'Assert' | 'AssertWithoutThrow' - | 'Sleep' - | 'WaitFor'; + | 'Sleep'; param: ParamType; } diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts index c003ec47..11973edb 100644 --- a/packages/web-integration/src/common/agent.ts +++ b/packages/web-integration/src/common/agent.ts @@ -120,7 +120,7 @@ export class PageAgent { async aiWaitFor(assertion: string, opt?: AgentWaitForOpt) { const { executor } = await this.taskExecutor.waitFor(assertion, { - timeoutMs: opt?.timeoutMs || 30 * 1000, + timeoutMs: opt?.timeoutMs || 15 * 1000, checkIntervalMs: opt?.checkIntervalMs || 3 * 1000, assertion, }); From 7775ac899a8bf8ebf460f9c4a884a1685f6d3885 Mon Sep 17 00:00:00 2001 From: yutao Date: Tue, 20 Aug 2024 14:33:57 +0800 Subject: [PATCH 05/26] feat: update docs for report --- .../docs/en/docs/getting-started/quick-start.mdx | 11 ++--------- .../docs/zh/docs/getting-started/quick-start.mdx | 15 +++------------ 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/apps/site/docs/en/docs/getting-started/quick-start.mdx b/apps/site/docs/en/docs/getting-started/quick-start.mdx index cf58a884..d7979ca0 100644 --- a/apps/site/docs/en/docs/getting-started/quick-start.mdx +++ b/apps/site/docs/en/docs/getting-started/quick-start.mdx @@ -113,8 +113,7 @@ npx playwright test ./e2e/ebay-search.spec.ts ### Step 5. view test report after running -After the above command executes successfully, it will output in the console: `Midscene - report file updated: xxx/midscene_run/report/xxx.html`. Open this file in a browser to view the report. - +After the above command executes successfully, the console will output: `Midscene - report file updated: ./current_cwd/midscene_run/report/some_id.html`. You can open this file in a browser to view the report. ## Integrate with Puppeteer @@ -203,18 +202,12 @@ npx ts-node demo.ts # ] ``` - ### Step 4: View the Run Report -After the above command executes successfully, the console will output: `Midscene - report file updated: xxx/midscene_run/report/xxx.html`. You can open this file in a browser to view the report. +After the above command executes successfully, the console will output: `Midscene - report file updated: ./current_cwd/midscene_run/report/some_id.html`. You can open this file in a browser to view the report. Alternatively, you can import the `./midscene_run/report/latest.web-dump.json` file into the [Visualization Tool](/visualization/) to view it. - -### View test report after running - -After running, Midscene will generate a log dump, which is placed in `./midscene_run/report/latest.web-dump.json` by default. Then put this file into [Visualization Tool](/visualization/), and you will have a clearer understanding of the process. - ## View demo report Click the 'Load Demo' button in the [Visualization Tool](/visualization/), you will be able to see the results of the previous code as well as some other samples. diff --git a/apps/site/docs/zh/docs/getting-started/quick-start.mdx b/apps/site/docs/zh/docs/getting-started/quick-start.mdx index ba91c63b..6c8a17a7 100644 --- a/apps/site/docs/zh/docs/getting-started/quick-start.mdx +++ b/apps/site/docs/zh/docs/getting-started/quick-start.mdx @@ -116,8 +116,7 @@ npx playwright test ./e2e/ebay-search.spec.ts ### Step 5. 查看测试报告 -当上面的命令执行成功后，会在控制台输出：`Midscene - report file updated: xxx/midscene_run/report/xxx.html` 通过浏览器打开该文件即可看到报告。 - +当上面的命令执行成功后，会在控制台输出：`Midscene - report file updated: ./current_cwd/midscene_run/report/some_id.html`，通过浏览器打开该文件即可看到报告。 ## 集成到 Puppeteer @@ -213,17 +212,9 @@ npx ts-node demo.ts ### 第四步：查看运行报告 -当上面的命令执行成功后，会在控制台输出：`Midscene - report file updated: xxx/midscene_run/report/xxx.html` 通过浏览器打开该文件即可看到报告。 - -也可以将 `./midscene_run/report/latest.web-dump.json` 文件导入 [可视化工具](/visualization/) 查看。 - - -## 访问示例报告 - -在 [可视化工具](/visualization/) 中，点击 `Load Demo` 按钮，你将能够看到上方代码的运行结果以及其他的一些示例。 - -![](/view-demo-visualization.gif) +当上面的命令执行成功后，会在控制台输出：`Midscene - report file updated: ./current_cwd/midscene_run/report/some_id.html`， 通过浏览器打开该文件即可看到报告。 +你也可以将 `./midscene_run/report/latest.web-dump.json` 文件导入 [可视化工具](/visualization/) 查看。 ## 查看示例报告 From d351b23fe585cbea43b5c731e96119c24784c948 Mon Sep 17 00:00:00 2001 From: yutao Date: Tue, 20 Aug 2024 18:59:35 +0800 Subject: [PATCH 06/26] feat: add 'wait-for' param in cli --- apps/site/docs/en/docs/usage/cli.md | 5 +-- apps/site/docs/zh/docs/usage/cli.md | 3 +- packages/cli/src/help.ts | 9 ++--- packages/cli/src/index.ts | 35 ++++++++++++++------ packages/web-integration/src/common/agent.ts | 8 +++-- packages/web-integration/src/common/tasks.ts | 1 - 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/apps/site/docs/en/docs/usage/cli.md b/apps/site/docs/en/docs/usage/cli.md index bc240b26..6e390d7d 100644 --- a/apps/site/docs/en/docs/usage/cli.md +++ b/apps/site/docs/en/docs/usage/cli.md @@ -57,12 +57,13 @@ Options: --help Display this help message --version Display the version -Actions (order matters, can be used multiple times): +Actions (the order matters, can be used multiple times): --action Perform an action, optional --assert Perform an assert, optional --query-output Save the result of the query to a file, this must be put before --query, optional --query Perform a query, optional - --sleep Sleep for a number of milliseconds, optional` + --wait-for Wait for a condition to be met. The timeout is set to 15 seconds. optional + --sleep Sleep for a number of milliseconds, optional ``` diff --git a/apps/site/docs/zh/docs/usage/cli.md b/apps/site/docs/zh/docs/usage/cli.md index 8619e5b2..2c7590e7 100644 --- a/apps/site/docs/zh/docs/usage/cli.md +++ b/apps/site/docs/zh/docs/usage/cli.md @@ -61,6 +61,7 @@ Actions (参数顺序很重要，可以支持多次使用): --assert Perform an assert, optional --query-output Save the result of the query to a file, this must be put before --query, optional --query Perform a query, optional + --wait-for Wait for a condition to be met. The timeout is set to 15 seconds. optional --sleep Sleep for a number of milliseconds, optional` ``` @@ -68,4 +69,4 @@ Actions (参数顺序很重要，可以支持多次使用): 1. Options 参数（任务信息）应始终放在 Actions 参数之前。 2. Actions 参数的顺序很重要。例如，`--action "某操作" --query "某数据"` 表示先执行操作，然后再查询。 -3. 如果有更复杂的需求，比如循环操作，使用 SDK 版本（而不是这个命令行工具）会更容易实现。 +3. 如果有更复杂的需求，比如循环操作，使用 SDK 版本（而不是这个命令行工具）会更合适。 diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.ts index cda349ff..8ac8ef65 100644 --- a/packages/cli/src/help.ts +++ b/packages/cli/src/help.ts @@ -17,23 +17,24 @@ if (process.argv.indexOf('--help') !== -1) { --assert Perform an assert, optional --query-output Save the result of the query to a file, this must be put before --query, optional --query Perform a query, optional + --wait-for Wait for a condition to be met. The timeout is set to 15 seconds. optional --sleep Sleep for a number of milliseconds, optional Examples: # headed mode (i.e. visible browser) to visit bing.com and search for 'weather today' - midscene --headed --url https://wwww.bing.com --action "type 'weather today' in search box, hit enter" --sleep 3000 + midscene --headed --url "https://wwww.bing.com" --action "type 'weather today' in search box, hit enter" --wait-for "there is weather info in the page" # visit github status page and save the status to ./status.json - midscene --url https://www.githubstatus.com/ \\ + midscene --url "https://www.githubstatus.com/" \\ --query-output status.json \\ --query '{name: string, status: string}[], service status of github page' Examples with Chinese Prompts # headed 模式（即可见浏览器）访问 baidu.com 并搜索“天气” - midscene --headed --url https://www.baidu.com --action "在搜索框输入 '天气', 敲回车" --sleep 3000 + midscene --headed --url "https://www.baidu.com" --action "在搜索框输入 '天气', 敲回车" --wait-for 界面上出现了天气信息 # 访问 Github 状态页面并将状态保存到 ./status.json - midscene --url https://www.githubstatus.com/ \\ + midscene --url "https://www.githubstatus.com/" \\ --query-output status.json \\ --query '{serviceName: string, status: string}[], github 页面的服务状态，返回服务名称' `); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index bab897d6..ae65fa13 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -52,6 +52,7 @@ const actionArgs = { queryOutput: 'query-output', query: 'query', sleep: 'sleep', + waitFor: 'wait-for', }; const defaultUA = @@ -113,21 +114,23 @@ Promise.resolve( const page = await browser.newPage(); await page.setUserAgent(ua); - await page.setViewport(viewportConfig); - updateSpin(stepString('launch', url)); - await page.goto(url); - updateSpin(stepString('waitForNetworkIdle', url)); - await page.waitForNetworkIdle(); - printStep('launched', url); - - const agent = new PuppeteerAgent(page); - let errorWhenRunning: Error | undefined; let argName: string; let argValue: ArgumentValueType; + let agent: PuppeteerAgent | undefined; try { + updateSpin(stepString('launch', url)); + await page.goto(url); + updateSpin(stepString('waitForNetworkIdle', url)); + await page.waitForNetworkIdle(); + printStep('launched', url); + + agent = new PuppeteerAgent(page, { + autoPrintReportMsg: false, + }); + let index = 0; let outputPath: string | undefined; let actionStarted = false; @@ -197,11 +200,23 @@ Promise.resolve( printStep(argName, String(argValue)); break; } + case actionArgs.waitFor: { + const param = arg.value; + assert(param, 'missing assertion for waitFor'); + assert(typeof param === 'string', 'assertion must be a string'); + await agent.aiWaitFor(param); + printStep(argName, String(argValue)); + break; + } } index += 1; } + printStep('Done', `report: ${agent.reportFile}`); } catch (e: any) { printStep(`${argName!} - Failed`, String(argValue!)); + if (agent?.reportFile) { + printStep('Report', agent.reportFile); + } printStep('Error', e.message); errorWhenRunning = e; } @@ -210,5 +225,3 @@ Promise.resolve( process.exit(errorWhenRunning ? 1 : 0); })(), ); - -// TODO: print report after running diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts index 11973edb..ee0ea628 100644 --- a/packages/web-integration/src/common/agent.ts +++ b/packages/web-integration/src/common/agent.ts @@ -1,3 +1,4 @@ +import { assert } from 'node:console'; import type { WebPage } from '@/common/page'; import type { AgentWaitForOpt, @@ -20,6 +21,8 @@ export interface PageAgentOpt { cache?: AiTaskCache; /* if auto generate report, default true */ generateReport?: boolean; + /* if auto print report msg, default true */ + autoPrintReportMsg?: boolean; } export class PageAgent { @@ -40,6 +43,7 @@ export class PageAgent { this.opts = Object.assign( { generateReport: true, + autoPrintReportMsg: true, groupName: 'Midscene Report', groupDescription: '', }, @@ -69,7 +73,7 @@ export class PageAgent { } writeOutActionDumps() { - const generateReport = this.opts.generateReport; + const { generateReport, autoPrintReportMsg } = this.opts; this.reportFile = writeLogFile({ fileName: this.reportFileName!, fileExt: groupedActionDumpFileExt, @@ -78,7 +82,7 @@ export class PageAgent { generateReport, }); - if (generateReport) { + if (generateReport && autoPrintReportMsg) { printReportMsg(this.reportFile); } } diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index bed13d1b..7880eeea 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -245,7 +245,6 @@ export class PageTaskExecutor { type: 'Action', subType: 'Hover', executor: async (param, { element }) => { - // console.log('executor args', param, element); assert(element, 'Element not found, cannot hover'); await this.page.mouse.move( element.center[0], From 0cc1e75526c4c1b110dd2a507f906461ab6e6896 Mon Sep 17 00:00:00 2001 From: yutao Date: Wed, 21 Aug 2024 16:13:17 +0800 Subject: [PATCH 07/26] fix: exactor for input --- .../tests/ai/puppeteer/showcase.test.ts | 134 ++++++++++-------- .../tests/ai/puppeteer/utils.ts | 3 +- .../tests/unit-test/fixtures/extractor.html | 55 ++++++- packages/web-integration/vitest.config.ts | 5 +- 4 files changed, 126 insertions(+), 71 deletions(-) diff --git a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts b/packages/web-integration/tests/ai/puppeteer/showcase.test.ts index c94e3611..4db7ac83 100644 --- a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts +++ b/packages/web-integration/tests/ai/puppeteer/showcase.test.ts @@ -3,27 +3,70 @@ import { describe, expect, it, vi } from 'vitest'; import { launchPage } from './utils'; vi.setConfig({ - testTimeout: 90 * 1000, + testTimeout: 180 * 1000, }); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -describe( - 'puppeteer integration', - () => { - it('Sauce Demo by Swag Lab', async () => { - const page = await launchPage('https://www.saucedemo.com/'); - const mid = new PuppeteerAgent(page); - - await mid.aiAction( - 'type "standard_user" in user name input, type "secret_sauce" in password, click "Login"', +describe('puppeteer integration', () => { + it.only('抖音来客', async () => { + const page = await launchPage('https://life-boe.bytedance.net/p/login', { + headless: false, + }); + const mid = new PuppeteerAgent(page); + + await mid.aiWaitFor('界面右侧有一个“手机号”输入框'); + await sleep(500 * 1000); + await mid.ai(` + 1、找到手机号码输入框，输入 12342512971 + 2、找到验证码输入框，输入 3518 + 3、找到“已阅读并同意”文本左边的 checkbox，执行点击操作 + 4、找到立即入驻按钮，执行点击操作 + `); + + await mid.aiWaitFor('界面上有一个“订单管理”按钮'); + }); + + it('Sauce Demo by Swag Lab', async () => { + const page = await launchPage('https://www.saucedemo.com/'); + const mid = new PuppeteerAgent(page); + + await mid.aiAction( + 'type "standard_user" in user name input, type "secret_sauce" in password, click "Login"', + ); + + await expect(async () => { + await mid.aiWaitFor('there is a cookie prompt in the UI', { + timeoutMs: 10 * 1000, + }); + }).rejects.toThrowError(); + + // find the items + const items = await mid.aiQuery( + '"{name: string, price: number, actionBtnName: string}[], return item name, price and the action button name on the lower right corner of each item (like "Remove")', + ); + console.log('item list', items); + expect(items.length).toBeGreaterThanOrEqual(2); + + await mid.aiAssert('The price of "Sauce Labs Onesie" is 7.99'); + }); + + it('extract the Github service status', async () => { + const page = await launchPage('https://www.githubstatus.com/'); + const mid = new PuppeteerAgent(page); + + const result = await mid.aiQuery( + 'this is a service status page. Extract all status data with this scheme: {[serviceName]: [statusText]}', + ); + console.log('Github service status', result); + + expect(async () => { + // there is no food delivery service on Github + await mid.aiAssert( + 'there is a "food delivery" service on page and is in normal state', ); - await expect(async () => { - await mid.aiWaitFor('there is a cookie prompt in the UI', { - timeoutMs: 10 * 1000, - }); - }).rejects.toThrowError(); + await sleep(2000); // find the items const items = await mid.aiQuery( @@ -34,53 +77,22 @@ describe( await mid.aiAssert('The price of "Sauce Labs Onesie" is 7.99'); }); + }); - it('extract the Github service status', async () => { - const page = await launchPage('https://www.githubstatus.com/'); - const mid = new PuppeteerAgent(page); - - const result = await mid.aiQuery( - 'this is a service status page. Extract all status data with this scheme: {[serviceName]: [statusText]}', - ); - console.log('Github service status', result); - - expect(async () => { - // there is no food delivery service on Github - await mid.aiAssert( - 'there is a "food delivery" service on page and is in normal state', - ); - - await sleep(2000); + it('extract the Github service status', async () => { + const page = await launchPage('https://www.githubstatus.com/'); + const mid = new PuppeteerAgent(page); - // find the items - const items = await mid.aiQuery( - '"{name: string, price: number, actionBtnName: string}[], return item name, price and the action button name on the lower right corner of each item (like "Remove")', - ); - console.log('item list', items); - expect(items.length).toBeGreaterThanOrEqual(2); + const result = await mid.aiQuery( + 'this is a service status page. Extract all status data with this scheme: {[serviceName]: [statusText]}', + ); + console.log('Github service status', result); - await mid.aiAssert('The price of "Sauce Labs Onesie" is 7.99'); - }); - }); - - it('extract the Github service status', async () => { - const page = await launchPage('https://www.githubstatus.com/'); - const mid = new PuppeteerAgent(page); - - const result = await mid.aiQuery( - 'this is a service status page. Extract all status data with this scheme: {[serviceName]: [statusText]}', + // obviously there is no food delivery service on Github + expect(async () => { + await mid.aiAssert( + 'there is a "food delivery" service on page and is in normal state', ); - console.log('Github service status', result); - - // obviously there is no food delivery service on Github - expect(async () => { - await mid.aiAssert( - 'there is a "food delivery" service on page and is in normal state', - ); - }).rejects.toThrowError(); - }); - }, - { - timeout: 90 * 1000, - }, -); + }).rejects.toThrowError(); + }); +}); diff --git a/packages/web-integration/tests/ai/puppeteer/utils.ts b/packages/web-integration/tests/ai/puppeteer/utils.ts index 4572c3e8..9f43d828 100644 --- a/packages/web-integration/tests/ai/puppeteer/utils.ts +++ b/packages/web-integration/tests/ai/puppeteer/utils.ts @@ -5,10 +5,11 @@ export async function launchPage( url: string, opt?: { viewport?: Viewport; + headless?: boolean; }, ) { const browser = await puppeteer.launch({ - // headless: false, + headless: opt?.headless, }); const page = await browser.newPage(); diff --git a/packages/web-integration/tests/unit-test/fixtures/extractor.html b/packages/web-integration/tests/unit-test/fixtures/extractor.html index 5cb71e18..a1619b71 100644 --- a/packages/web-integration/tests/unit-test/fixtures/extractor.html +++ b/packages/web-integration/tests/unit-test/fixtures/extractor.html @@ -97,17 +97,60 @@

Form

alt="small_img" />
-
+ + - + _midscene_retrieve_task_id="36bc72f23a"> + + + + + + + + + +
+ +
+ + +
+ + + + + + + - + + + +
@@ -157,7 +174,7 @@

Form

+ + @@ -148,6 +142,7 @@

Form

class="life-core-input-inner__wrapper life-core-input-inner__wrapper-border life-core-input-inner__wrapper-size-md life-core-input-inner__wrapper-add-suffix"> +