diff --git a/apps/site/docs/en/docs/usage/cache.md b/apps/site/docs/en/docs/usage/cache.md index 0adc43bc..212a4145 100644 --- a/apps/site/docs/en/docs/usage/cache.md +++ b/apps/site/docs/en/docs/usage/cache.md @@ -1,10 +1,10 @@ # Cache -Midscene.js provides AI caching capabilities to enhance the stability and speed of the entire AI execution process. The cache here mainly refers to caching the elements recognized by AI on the page. When the page elements have not changed, the AI's query results will be cached. +Midscene.js provides AI caching capabilities to improve the stability and speed of the entire AI execution process. The cache here mainly refers to caching AI's recognition of page elements. When the page elements have not changed, the AI query results will be cached. ## Instructions -Currently, the caching capability is only supported on `Playwright`, and Midscene can support caching at the test suite level. +Currently, the caching capability is supported in all scenarios, and Midscene can support file-level caching. **Usage** @@ -18,41 +18,39 @@ Currently, the caching capability is only supported on `Playwright`, and Midscen * **before** ![](/cache/no-cache-time.png) - * **after** ![](/cache/use-cache-time.png) - ## Cache Content -Currently, Midscene's caching strategy on Playwright is mainly based on test suites, and AI behaviors within each test suite will be cached. The cache content mainly includes two types: +Currently, Midscene's caching strategy in all scenarios is mainly based on the test file unit. AI behavior in each test file will be cached. The cached content is mainly divided into two categories: -* AI task planning (planning is the result of ai and ai action methods) -* AI element recognition +* AI's planning for tasks (Planning, i.e., the results of ai and aiAction methods) +* AI's recognition of elements -The content of `aiQuery` will not be cached, so `aiQuery` can be used to verify whether the previous AI tasks meet expectations. +The content of `aiQuery` will not be cached, so you can use `aiQuery` to confirm whether the previous AI tasks meet expectations. **Task Planning** ```js -await ai("Move the mouse to the second task, then click the delete button on the right of the task"); +await ai("Move the mouse to the second task and click the delete button on the right side of the task"); ``` -The above task planning will be broken down into: +The above task planning will be decomposed into: ```js Hover: Move the mouse to the second task "Learn JS today" -Click: Click the delete button on the right of the task "Learn JS today" +Click: Click the delete button on the right side of the task "Learn JS today" ``` -When the page URL and dimensions have not changed, the results of the above tasks will be directly cached when caching is enabled. +When the URL address and page width and height have not changed, enabling the cache will directly cache the results of the above tasks. **Element Recognition** -After AI has planned the tasks based on the user's instructions, it needs to operate on specific elements, which requires AI's ability to recognize page elements. For example, the following task: +After the AI has planned the user's instructions into tasks, it needs to operate on specific elements, so the AI's element recognition capability is needed. For example, the following task: ```js Hover: Move the mouse to the second task "Learn JS today" @@ -68,9 +66,9 @@ Width: 100 Height: 30 ``` -## Caching Strategy +## Cache Strategy -When using the `MIDSCENE_CACHE=true` environment variable, caching will automatically be performed according to the test suites in `Playwright`: +When using the `MIDSCENE_CACHE=true` environment variable, caching will be automatically performed according to Playwright's test groups: ```ts // todo-mvc.spec.ts @@ -82,127 +80,128 @@ test.beforeEach(async ({ page }) => { }); test('ai todo', async ({ page, ai, aiQuery }) => { - await ai("Enter 'Learn JS today' in the task box, then press Enter"); + await ai("Enter \"Learn JS today\" in the task box, then press Enter to create"); }); test('ai todo2', async ({ page, ai, aiQuery }) => { - await ai("Enter 'Learn JS today' in the task box, then press Enter"); + await ai("Enter \"Learn JS today\" in the task box, then press Enter to create"); }); ``` -The above `test` will generate caches according to the dimensions of `ai todo` and `ai todo2`, and cache files `todo-mvc.spec:10(ai todo).json` and `todo-mvc.spec:13(ai todo2).json` will be generated in the `midscene/midscene_run/cache` directory at the root of the project. +The above `test` will generate caches along the dimensions of `ai todo` and `ai todo2`, and `todo-mvc.spec.ts-1.json` and `todo-mvc.spec.ts-2.json` cache files will be generated in the `midscene/midscene_run/cache` directory in the project root. -**Cache File Description** +**Cache File Introduction** ```json { "pkgName": "@midscene/web", - // Current midscene version + // The midscene version currently in use "pkgVersion": "0.1.2", - // Test file address and line number - "taskFile": "todo-mvc.spec.ts:10", - // Test task title - "taskTitle": "ai todo", + // Test file address and index + "cacheId": "tests/ai/e2e/ai-auto-todo.spec.ts-1", "aiTasks": [ { - // Task type, currently only plan and locate - // plan is determined by AI based on user's task - "type": "plan", - "pageContext": { - // URL when AI executes the task - "url": "https://todomvc.com/examples/react/dist/", - // Page dimensions - "size": { - "width": 1280, - "height": 720 - } - }, // User's prompt instruction - "prompt": "Enter 'Learn JS today' in the task box, then press Enter to create", - "response": { - // AI's tasks - "plans": [ - { - "thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.", - "type": "Locate", - "param": { - "prompt": "The input box with the placeholder text 'What needs to be done?'." + "prompt": "Enter \"Learn JS today\" in the task box, then press Enter to create", + "tasks": [ + { + // Task type, currently only plan and locate + // plan is determined by AI based on user's task + "type": "plan", + "pageContext": { + // Address when AI executes tasks + "url": "https://todomvc.com/examples/react/dist/", + // Page width and height + "size": { + "width": 1280, + "height": 720 } }, - { - "thought": "Once the input box is located, we need to enter the task description.", - "type": "Input", - "param": { - "value": "Learn JS today" + // User's prompt instruction + "prompt": "Enter \"Learn JS today\" in the task box, then press Enter to create", + "response": { + // AI's tasks + "plans": [ + { + "thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.", + "type": "Locate", + "param": { + "prompt": "The input box with the placeholder text 'What needs to be done?'." + } + }, + { + "thought": "Once the input box is located, we need to enter the task description.", + "type": "Input", + "param": { + "value": "Learn JS today" + } + }, + { + "thought": "After entering the task, we need to commit it by pressing 'Enter'.", + "type": "KeyboardPress", + "param": { + "value": "Enter" + } + } + ] + } + }, + { + // locate is to find a specific element + "type": "locate", + "pageContext": { + // Address when AI executes tasks + "url": "https://todomvc.com/examples/react/dist/", + // Page width and height + "size": { + "width": 1280, + "height": 720 } }, - { - "thought": "After entering the task, we need to commit it by pressing 'Enter'.", - "type": "KeyboardPress", - "param": { - "value": "Enter" - } + // User's prompt instruction + "prompt": "The input box with the placeholder text 'What needs to be done?'.", + "response": { + // Returned element content + "elements": [ + { + // Why AI found this element + "reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.", + // Element text + "text": "What needs to be done?", + // Unique ID generated based on the element (generated based on position and size) + "id": "3530a9c1eb" + } + ], + "errors": [] } - ] - } - }, - { - // locate is for finding specific elements - "type": "locate", - "pageContext": { - // URL when AI executes the task - "url": "https://todomvc.com/examples/react/dist/", - // Page dimensions - "size": { - "width": 1280, - "height": 720 } - }, - // User's prompt instruction - "prompt": "The input box with the placeholder text 'What needs to be done?'.", - "response": { - // Returned element content - "elements": [ - { - // Why AI found this element - "reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.", - // Element text - "text": "What needs to be done?", - // Unique ID generated based on the element (based on position and size) - "id": "3530a9c1eb" - } - ], - "errors": [] - } - } + ] ] //... } ``` -When the `MIDSCENE_CACHE=true` environment variable is used and there are cache files, the corresponding AI results will be read from the above cache files. The conditions for cache hits are as follows: +When the `MIDSCENE_CACHE=true` environment variable is used and there are cache files, the AI's corresponding results will be read through the above cache file. The following are the conditions for cache hit: 1. The same test file and test title -2. The same Midscene package name, version, and previous tasks -3. The same page URL and dimensions when executing the task -4. The current page contains the exact same elements as last time (only required for element locating tasks) +2. Midscene package name, version, and last task are consistent +3. The page address and page width and height where the corresponding task is executed are consistent +4. The current page has exactly the same elements as last time (only required for locate element tasks) -## Frequently Asked Questions +## Common Issues -### Why provide caching capabilities? +### Why provide caching capability? -Caching capabilities mainly solve the following problems: +The caching capability mainly solves the following problems: -1. High AI response latency: A task can take several seconds. When there are dozens or even hundreds of tasks, it can be very time-consuming. -2. AI response stability: Through tuning and experimentation, we found that GPT-4 has over 90% accuracy in page element recognition tasks, but it still cannot reach 100% accuracy. Caching capabilities can effectively reduce online stability issues. +1. High AI response latency, a task will take several seconds, and when there are dozens or even hundreds of tasks, there will be a higher latency +2. AI response stability, through training and experiments, we found that GPT-4 has an accuracy rate of over 95% in page element recognition tasks, but it cannot reach 100% accuracy yet. The caching capability can effectively reduce online stability issues ### What happens if the cache is not hit? -For AI behaviors that do not hit the cache, AI will re-execute the task, and the cache will be updated after the entire test suite execution is completed. You can check the cache files to determine which tasks have been updated. +For AI behaviors that do not hit the cache, they will be re-executed by AI, and the cache will be updated after the entire test group is executed. You can check the cache file to determine which tasks have been updated. ### How to manually remove the cache? -* Deleting the corresponding cache files will automatically invalidate the entire test suite's cache. -* Deleting specific tasks in the cache file will automatically invalidate the corresponding tasks. After the task is successfully executed, the task will be updated. Deleting previous tasks will not affect subsequent tasks. - - +* When deleting the corresponding cache file, the cache of the entire test group will automatically become invalid +* When deleting specific tasks in the cache file, the corresponding tasks will automatically become invalid. Deleting the tasks before will not affect the tasks after. The tasks will be updated after successful execution \ No newline at end of file diff --git a/apps/site/docs/zh/docs/usage/cache.md b/apps/site/docs/zh/docs/usage/cache.md index b5b34543..d2272584 100644 --- a/apps/site/docs/zh/docs/usage/cache.md +++ b/apps/site/docs/zh/docs/usage/cache.md @@ -4,7 +4,7 @@ Midscene.js 提供了 AI 缓存能力，用于提升整个 AI 执行过程的稳 ## 使用说明 -目前缓存的能力仅在 `Playwright` 上进行了支持，Midscene 能够支持测试组级别的缓存。 +目前缓存的能力在所有场景下都进行了支持，Midscene 能够支持文件级别的缓存。 **使用方式** @@ -26,7 +26,7 @@ Midscene.js 提供了 AI 缓存能力，用于提升整个 AI 执行过程的稳 ## 缓存内容 -目前 Midscene 在 Playwright 上的缓存策略主要是以测试组为单位，在每个测试组里的 AI 行为将发生缓存。目前缓存的内容主要有两类： +目前 Midscene 在所有场景下的缓存策略主要是以测试文件为单位，在每个测试文件里的 AI 行为将发生缓存。目前缓存的内容主要有两类： * AI 对于任务的规划（Planning, 即 ai 和 aiAction 方法的结果） * AI 对于元素的识别 @@ -88,7 +88,7 @@ test('ai todo2', async ({ page, ai, aiQuery }) => { }); ``` -上面的 `test` 将按照 `ai todo` 和 `ai todo2` 这两个维度产生缓存，分别会在项目的根目录中的 `midscene/midscene_run/cache` 中生成 `todo-mvc.spec:10(ai todo).json` 和 `todo-mvc.spec:13(ai todo2).json` 缓存文件。 +上面的 `test` 将按照 `ai todo` 和 `ai todo2` 这两个维度产生缓存，分别会在项目的根目录中的 `midscene/midscene_run/cache` 中生成 `todo-mvc.spec.ts-1.json` 和 `todo-mvc.spec.ts-2.json` 缓存文件。 **缓存文件介绍** @@ -97,82 +97,85 @@ test('ai todo2', async ({ page, ai, aiQuery }) => { "pkgName": "@midscene/web", // 当前使用的 midscene 版本 "pkgVersion": "0.1.2", - // 测试文件地址和行数 - "taskFile": "todo-mvc.spec.ts:10", - // 测试任务标题 - "taskTitle": "ai todo", + // 测试文件地址和下标 + "cacheId": "tests/ai/e2e/ai-auto-todo.spec.ts-1", "aiTasks": [ { - // 任务类型，目前只有 plan 和 locate - // plan 为 AI 通过用户的任务决定 - "type": "plan", - "pageContext": { - // AI 执行任务时的地址 - "url": "https://todomvc.com/examples/react/dist/", - // 页面宽高 - "size": { - "width": 1280, - "height": 720 - } - }, // 用户的 prompt 指令 "prompt": "Enter \"Learn JS today\" in the task box, then press Enter to create", - "response": { - // AI 的任务 - "plans": [ - { - "thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.", - "type": "Locate", - "param": { - "prompt": "The input box with the placeholder text 'What needs to be done?'." + "tasks": [ + { + // 任务类型，目前只有 plan 和 locate + // plan 为 AI 通过用户的任务决定 + "type": "plan", + "pageContext": { + // AI 执行任务时的地址 + "url": "https://todomvc.com/examples/react/dist/", + // 页面宽高 + "size": { + "width": 1280, + "height": 720 } }, - { - "thought": "Once the input box is located, we need to enter the task description.", - "type": "Input", - "param": { - "value": "Learn JS today" + // 用户的 prompt 指令 + "prompt": "Enter \"Learn JS today\" in the task box, then press Enter to create", + "response": { + // AI 的任务 + "plans": [ + { + "thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.", + "type": "Locate", + "param": { + "prompt": "The input box with the placeholder text 'What needs to be done?'." + } + }, + { + "thought": "Once the input box is located, we need to enter the task description.", + "type": "Input", + "param": { + "value": "Learn JS today" + } + }, + { + "thought": "After entering the task, we need to commit it by pressing 'Enter'.", + "type": "KeyboardPress", + "param": { + "value": "Enter" + } + } + ] + } + }, + { + // locate 为需要查找特定元素 + "type": "locate", + "pageContext": { + // AI 执行任务时的地址 + "url": "https://todomvc.com/examples/react/dist/", + // 页面的宽高 + "size": { + "width": 1280, + "height": 720 } }, - { - "thought": "After entering the task, we need to commit it by pressing 'Enter'.", - "type": "KeyboardPress", - "param": { - "value": "Enter" - } + // 用户的 prompt 指令 + "prompt": "The input box with the placeholder text 'What needs to be done?'.", + "response": { + // 返回的元素内容 + "elements": [ + { + // AI 为什么找到了这个元素 + "reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.", + // 元素的文本 + "text": "What needs to be done?", + // 基于元素生成的唯一 ID（基于位置和大小生成） + "id": "3530a9c1eb" + } + ], + "errors": [] } - ] - } - }, - { - // locate 为需要查找特定元素 - "type": "locate", - "pageContext": { - // AI 执行任务时的地址 - "url": "https://todomvc.com/examples/react/dist/", - // 页面的宽高 - "size": { - "width": 1280, - "height": 720 } - }, - // 用户的 prompt 指令 - "prompt": "The input box with the placeholder text 'What needs to be done?'.", - "response": { - // 返回的元素内容 - "elements": [ - { - // AI 为什么找到了这个元素 - "reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.", - // 元素的文本 - "text": "What needs to be done?", - // 基于元素生成的唯一 ID（基于位置和大小生成） - "id": "3530a9c1eb" - } - ], - "errors": [] - } - } + ] ] //... } @@ -192,7 +195,7 @@ test('ai todo2', async ({ page, ai, aiQuery }) => { 缓存能力主要解决了以下问题： 1. AI 响应延迟高，一个任务将会耗费几秒钟，当有几十条甚至几百条任务时将会有较高的耗时 -2. AI 响应稳定性，通过调教和实验中我们发现 GPT-4 在页面元素识别的任务上有 90%+ 的准确率，但尚无法达到 100% 的准确率，通过缓存能力能够有效降低线上稳定性问题 +2. AI 响应稳定性，通过调教和实验中我们发现 GPT-4 在页面元素识别的任务上有 95%+ 的准确率，但尚无法达到 100% 的准确率，通过缓存能力能够有效降低线上稳定性问题 ### 未命中缓存会发生什么？ diff --git a/packages/midscene/src/action/executor.ts b/packages/midscene/src/action/executor.ts index 19472e7e..5c90262a 100644 --- a/packages/midscene/src/action/executor.ts +++ b/packages/midscene/src/action/executor.ts @@ -7,7 +7,7 @@ import type { ExecutionTaskReturn, ExecutorContext, } from '@/types'; -import { getPkgInfo } from '@/utils'; +import { getMidscenePkgInfo } from '@midscene/shared/fs'; export class Executor { name: string; @@ -176,7 +176,7 @@ export class Executor { dump(): ExecutionDump { const dumpData: ExecutionDump = { - sdkVersion: getPkgInfo().version, + sdkVersion: getMidscenePkgInfo(__dirname).version, logTime: Date.now(), name: this.name, description: this.description, diff --git a/packages/midscene/src/insight/utils.ts b/packages/midscene/src/insight/utils.ts index 9d6de620..ec2bc7da 100644 --- a/packages/midscene/src/insight/utils.ts +++ b/packages/midscene/src/insight/utils.ts @@ -17,11 +17,11 @@ import type { } from '@/types'; import { getLogDir, - getPkgInfo, insightDumpFileExt, stringifyDumpData, writeLogFile, } from '@/utils'; +import { getMidscenePkgInfo } from '@midscene/shared/fs'; let logFileName = ''; const logContent: string[] = []; @@ -39,7 +39,7 @@ export function writeInsightDump( const id = logId || randomUUID(); const baseData: DumpMeta = { - sdkVersion: getPkgInfo().version, + sdkVersion: getMidscenePkgInfo(__dirname).version, logTime: Date.now(), }; const finalData: InsightDump = { diff --git a/packages/midscene/src/utils.ts b/packages/midscene/src/utils.ts index e9ecd2d2..d1b037df 100644 --- a/packages/midscene/src/utils.ts +++ b/packages/midscene/src/utils.ts @@ -2,7 +2,8 @@ import assert from 'node:assert'; import { randomUUID } from 'node:crypto'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import path, { basename, dirname, join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; +import { getMidscenePkgInfo } from '@midscene/shared/fs'; import type { Rect, ReportDumpWithAttributes } from './types'; interface PkgInfo { @@ -11,28 +12,7 @@ interface PkgInfo { dir: string; } -let pkg: PkgInfo | undefined; -export function getPkgInfo(): PkgInfo { - if (pkg) { - return pkg; - } - - const pkgDir = findNearestPackageJson(__dirname); - assert(pkgDir, 'package.json not found'); - const pkgJsonFile = join(pkgDir, 'package.json'); - - if (pkgJsonFile) { - const { name, version } = JSON.parse(readFileSync(pkgJsonFile, 'utf-8')); - pkg = { name, version, dir: pkgDir }; - return pkg; - } - return { - name: 'midscene-unknown-page-name', - version: '0.0.0', - dir: pkgDir, - }; -} - +const midscenePkgInfo = getMidscenePkgInfo(__dirname); let logDir = join(process.cwd(), './midscene_run/'); let logEnvReady = false; export const insightDumpFileExt = 'insight-dump.json'; @@ -58,7 +38,7 @@ export function writeDumpReport( fileName: string, dumpData: string | ReportDumpWithAttributes[], ) { - const { dir } = getPkgInfo(); + const { dir } = midscenePkgInfo; const reportTplPath = join(dir, './report/index.html'); existsSync(reportTplPath) || assert(false, `report template not found: ${reportTplPath}`); @@ -141,7 +121,7 @@ export function writeLogFile(opts: { } export function getTmpDir() { - const path = join(tmpdir(), getPkgInfo().name); + const path = join(tmpdir(), midscenePkgInfo.name); mkdirSync(path, { recursive: true }); return path; } @@ -180,25 +160,3 @@ export function replacerForPageObject(key: string, value: any) { export function stringifyDumpData(data: any, indents?: number) { return JSON.stringify(data, replacerForPageObject, indents); } - -/** - * Find the nearest package.json file recursively - * @param {string} dir - Home directory - * @returns {string|null} - The most recent package.json file path or null - */ -export function findNearestPackageJson(dir: string): string | null { - const packageJsonPath = path.join(dir, 'package.json'); - - if (existsSync(packageJsonPath)) { - return dir; - } - - const parentDir = path.dirname(dir); - - // Return null if the root directory has been reached - if (parentDir === dir) { - return null; - } - - return findNearestPackageJson(parentDir); -} diff --git a/packages/midscene/tests/ai/inspector/test-data/visualstudio/output.png b/packages/midscene/tests/ai/inspector/test-data/visualstudio/output.png index 4bf5d9dc..044e69d8 100644 Binary files a/packages/midscene/tests/ai/inspector/test-data/visualstudio/output.png and b/packages/midscene/tests/ai/inspector/test-data/visualstudio/output.png differ diff --git a/packages/midscene/tests/ai/inspector/test-data/visualstudio/output_without_text.png b/packages/midscene/tests/ai/inspector/test-data/visualstudio/output_without_text.png index 1ce921fd..f0b911a1 100644 Binary files a/packages/midscene/tests/ai/inspector/test-data/visualstudio/output_without_text.png and b/packages/midscene/tests/ai/inspector/test-data/visualstudio/output_without_text.png differ diff --git a/packages/shared/modern.config.ts b/packages/shared/modern.config.ts index 8bf0dfb8..284d36a5 100644 --- a/packages/shared/modern.config.ts +++ b/packages/shared/modern.config.ts @@ -9,7 +9,9 @@ export default defineConfig({ index: './src/index.ts', img: './src/img/index.ts', constants: './src/constants/index.ts', + fs: './src/fs/index.ts', }, target: 'es2017', + externals: ['node:buffer'], }, }); diff --git a/packages/shared/package.json b/packages/shared/package.json index 6dc909cf..f44133ab 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -15,6 +15,11 @@ "import": "./dist/es/constants.js", "require": "./dist/lib/constants.js" }, + "./fs": { + "types": "./src/fs/index.ts", + "import": "./dist/es/fs.js", + "require": "./dist/lib/fs.js" + }, "./img": { "types": "./src/img/index.ts", "import": "./dist/es/img.js", @@ -25,7 +30,8 @@ "*": { ".": ["./src/index.ts"], "constants": ["./src/constants/index.ts"], - "img": ["./src/img/index.ts"] + "img": ["./src/img/index.ts"], + "fs": ["./src/fs/index.ts"] } }, "files": ["dist", "src", "README.md"], @@ -77,13 +83,19 @@ "types": "./dist/types/img.d.ts", "import": "./dist/es/img.js", "require": "./dist/lib/img.js" + }, + "./fs": { + "types": "./dist/types/fs.d.ts", + "import": "./dist/es/fs.js", + "require": "./dist/lib/fs.js" } }, "typesVersions": { "*": { ".": ["./dist/types/index.d.ts"], "constants": ["./dist/types/constants.d.ts"], - "img": ["./dist/types/img.d.ts"] + "img": ["./dist/types/img.d.ts"], + "fs": ["./dist/types/fs.d.ts"] } } } diff --git a/packages/shared/src/fs/index.ts b/packages/shared/src/fs/index.ts new file mode 100644 index 00000000..ba7b7729 --- /dev/null +++ b/packages/shared/src/fs/index.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert'; +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +interface PkgInfo { + name: string; + version: string; + dir: string; +} + +let pkg: PkgInfo | undefined; +export function getMidscenePkgInfo(dir: string): PkgInfo { + if (pkg) { + return pkg; + } + + const pkgDir = findNearestPackageJson(dir || process.cwd()); + assert(pkgDir, 'package.json not found'); + const pkgJsonFile = join(pkgDir, 'package.json'); + + if (pkgJsonFile) { + const { name, version } = JSON.parse(readFileSync(pkgJsonFile, 'utf-8')); + pkg = { name, version, dir: pkgDir }; + return pkg; + } + return { + name: 'midscene-unknown-page-name', + version: '0.0.0', + dir: pkgDir, + }; +} + +/** + * Find the nearest package.json file recursively + * @param {string} dir - Home directory + * @returns {string|null} - The most recent package.json file path or null + */ +export function findNearestPackageJson(dir: string): string | null { + const packageJsonPath = join(dir, 'package.json'); + if (existsSync(packageJsonPath)) { + return dir; + } + + const parentDir = dirname(dir); + + // Return null if the root directory has been reached + if (parentDir === dir) { + return null; + } + + return findNearestPackageJson(parentDir); +} diff --git a/packages/web-integration/modern.config.ts b/packages/web-integration/modern.config.ts index b567bb28..c4001f3c 100644 --- a/packages/web-integration/modern.config.ts +++ b/packages/web-integration/modern.config.ts @@ -14,5 +14,6 @@ export default defineConfig({ 'playwright-report': './src/playwright/reporter/index.ts', }, target: 'es2017', + externals: ['@midscene/core', 'node:fs'], }, }); diff --git a/packages/web-integration/package.json b/packages/web-integration/package.json index e009f6c7..d0d5b68f 100644 --- a/packages/web-integration/package.json +++ b/packages/web-integration/package.json @@ -62,7 +62,10 @@ "build:watch": "modern build -w -c ./modern.config.ts & modern build -w -c ./modern.inspect.config.ts", "test": "vitest --run", "test:u": "vitest --run -u", - "test:ai": "AITEST=true npm run test", + "test:ai": "AI_TEST_TYPE=web npm run test", + "test:ai:cache": "MIDSCENE_CACHE=true AI_TEST_TYPE=web npm run test", + "test:ai:all": "npm run test:ai:web && npm run test:ai:native", + "test:ai:native": "MIDSCENE_CACHE=true AI_TEST_TYPE=native npm run test", "new": "modern new", "upgrade": "modern upgrade", "prepublishOnly": "npm run build", diff --git a/packages/web-integration/playwright.config.ts b/packages/web-integration/playwright.config.ts index 3e7374ea..0b81652f 100644 --- a/packages/web-integration/playwright.config.ts +++ b/packages/web-integration/playwright.config.ts @@ -45,12 +45,12 @@ export default defineConfig({ MIDSCENE_REPORT ? { name: 'report', - testDir: './tests/ai/playright-report', + testDir: './tests/ai/web/playwright-report-test', use: { ...devices['Desktop Chrome'] }, } : { name: 'e2e', - testDir: './tests/ai/playright', + testDir: './tests/ai/web/playwright', use: { ...devices['Desktop Chrome'] }, }, ], diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts index f18a022e..f3948316 100644 --- a/packages/web-integration/src/common/agent.ts +++ b/packages/web-integration/src/common/agent.ts @@ -16,6 +16,7 @@ import { printReportMsg, reportFileName } from './utils'; export interface PageAgentOpt { testId?: string; + cacheId?: string; groupName?: string; groupDescription?: string; cache?: AiTaskCache; @@ -55,7 +56,7 @@ export class PageAgent { executions: [], }; this.taskExecutor = new PageTaskExecutor(this.page, { - cache: opts?.cache || { aiTasks: [] }, + cacheId: opts?.cacheId, }); this.reportFileName = reportFileName(opts?.testId || 'web'); } diff --git a/packages/web-integration/src/common/task-cache.ts b/packages/web-integration/src/common/task-cache.ts index ed99e2d1..c0093e7d 100644 --- a/packages/web-integration/src/common/task-cache.ts +++ b/packages/web-integration/src/common/task-cache.ts @@ -1,5 +1,13 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; import type { AIElementParseResponse, PlanningAction } from '@midscene/core'; -import type { WebUIContext } from './utils'; +import { + getLogDirByType, + stringifyDumpData, + writeLogFile, +} from '@midscene/core/utils'; +import { getMidscenePkgInfo } from '@midscene/shared/fs'; +import { type WebUIContext, generateCacheId } from './utils'; export type PlanTask = { type: 'plan'; @@ -30,21 +38,71 @@ export type LocateTask = { export type AiTasks = Array; export type AiTaskCache = { - aiTasks: AiTasks; + aiTasks: Array<{ + prompt: string; + tasks: AiTasks; + }>; }; export class TaskCache { - cache: AiTaskCache | undefined; + cache: AiTaskCache; + + cacheId: string; newCache: AiTaskCache; - constructor(opts?: { cache: AiTaskCache }) { - this.cache = opts?.cache; + midscenePkgInfo: ReturnType; + + constructor(opts?: { fileName?: string }) { + this.midscenePkgInfo = getMidscenePkgInfo(__dirname); + this.cacheId = generateCacheId(opts?.fileName); + this.cache = this.readCacheFromFile() || { + aiTasks: [], + }; this.newCache = { aiTasks: [], }; } + getCacheGroupByPrompt(aiActionPrompt: string) { + const { aiTasks = [] } = this.cache || { aiTasks: [] }; + const index = aiTasks.findIndex((item) => item.prompt === aiActionPrompt); + const newCacheGroup: AiTasks = []; + this.newCache.aiTasks.push({ + prompt: aiActionPrompt, + tasks: newCacheGroup, + }); + return { + readCache: ( + pageContext: WebUIContext, + type: T, + actionPrompt: string, + ) => { + if (index === -1) { + return false; + } + if (type === 'plan') { + return this.readCache( + pageContext, + type, + actionPrompt, + aiTasks[index].tasks, + ) as T extends 'plan' ? PlanTask['response'] : LocateTask['response']; + } + return this.readCache( + pageContext, + type, + actionPrompt, + aiTasks[index].tasks, + ) as T extends 'plan' ? PlanTask['response'] : LocateTask['response']; + }, + saveCache: (cache: PlanTask | LocateTask) => { + newCacheGroup.push(cache); + this.writeCacheToFile(); + }, + }; + } + /** * Read and return cached responses asynchronously based on specific criteria * This function is mainly used to read cached responses from a certain storage medium. @@ -65,26 +123,28 @@ export class TaskCache { pageContext: WebUIContext, type: 'plan', userPrompt: string, + cacheGroup: AiTasks, ): PlanTask['response']; readCache( pageContext: WebUIContext, type: 'locate', userPrompt: string, + cacheGroup: AiTasks, ): LocateTask['response']; readCache( pageContext: WebUIContext, type: 'plan' | 'locate', userPrompt: string, + cacheGroup: AiTasks, ): PlanTask['response'] | LocateTask['response'] | false { - if (this.cache) { - const { aiTasks } = this.cache; - const index = aiTasks.findIndex((item) => item.prompt === userPrompt); + if (cacheGroup.length > 0) { + const index = cacheGroup.findIndex((item) => item.prompt === userPrompt); if (index === -1) { return false; } - const taskRes = aiTasks.splice(index, 1)[0]; + const taskRes = cacheGroup.splice(index, 1)[0]; // The corresponding element cannot be found in the new context if ( @@ -113,12 +173,6 @@ export class TaskCache { return false; } - saveCache(cache: PlanTask | LocateTask) { - if (cache) { - this.newCache?.aiTasks.push(cache); - } - } - pageContextEqual( taskPageContext: LocateTask['pageContext'], pageContext: WebUIContext, @@ -139,4 +193,42 @@ export class TaskCache { generateTaskCache() { return this.newCache; } + + readCacheFromFile() { + const cacheFile = join(getLogDirByType('cache'), `${this.cacheId}.json`); + if (process.env.MIDSCENE_CACHE === 'true' && existsSync(cacheFile)) { + try { + const data = readFileSync(cacheFile, 'utf8'); + const jsonData = JSON.parse(data); + if ( + jsonData.pkgName !== this.midscenePkgInfo.name || + jsonData.pkgVersion !== this.midscenePkgInfo.version + ) { + return undefined; + } + return jsonData as AiTaskCache; + } catch (err) { + return undefined; + } + } + return undefined; + } + + writeCacheToFile() { + const midscenePkgInfo = getMidscenePkgInfo(__dirname); + writeLogFile({ + fileName: `${this.cacheId}`, + fileExt: 'json', + fileContent: stringifyDumpData( + { + pkgName: midscenePkgInfo.name, + pkgVersion: midscenePkgInfo.version, + cacheId: this.cacheId, + ...this.newCache, + }, + 2, + ), + type: 'cache', + }); + } } diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index dc37e0e9..f227e3f4 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -29,7 +29,7 @@ import { base64Encoded } from '@midscene/shared/img'; import type { KeyInput } from 'puppeteer'; import type { ElementInfo } from '../extractor'; import type { WebElementInfo } from '../web-element'; -import { type AiTaskCache, TaskCache } from './task-cache'; +import { TaskCache } from './task-cache'; import { type WebUIContext, parseContextFromWebPage } from './utils'; interface ExecutionResult { @@ -44,12 +44,15 @@ export class PageTaskExecutor { taskCache: TaskCache; - constructor(page: WebPage, opts: { cache: AiTaskCache }) { + constructor(page: WebPage, opts: { cacheId: string | undefined }) { this.page = page; this.insight = new Insight(async () => { return await parseContextFromWebPage(page); }); - this.taskCache = new TaskCache(opts); + + this.taskCache = new TaskCache({ + fileName: opts?.cacheId, + }); } private async recordScreenshot(timing: ExecutionRecorderItem['timing']) { @@ -91,7 +94,10 @@ export class PageTaskExecutor { return taskWithScreenshot; } - private async convertPlanToExecutable(plans: PlanningAction[]) { + private async convertPlanToExecutable( + plans: PlanningAction[], + cacheGroup?: ReturnType, + ) { const tasks: ExecutionTaskApply[] = plans .map((plan) => { if (plan.type === 'Locate') { @@ -107,7 +113,7 @@ export class PageTaskExecutor { }; this.insight.onceDumpUpdatedFn = dumpCollector; const pageContext = await this.insight.contextRetrieverFn(); - const locateCache = this.taskCache.readCache( + const locateCache = cacheGroup?.readCache( pageContext, 'locate', param.prompt, @@ -127,7 +133,7 @@ export class PageTaskExecutor { }); if (locateResult) { - this.taskCache.saveCache({ + cacheGroup?.saveCache({ type: 'locate', pageContext: { url: pageContext.url, @@ -337,7 +343,7 @@ export class PageTaskExecutor { userPrompt: string /* , actionInfo?: { actionType?: EventActions[number]['action'] } */, ): Promise { const taskExecutor = new Executor(userPrompt); - + const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt); let plans: PlanningAction[] = []; const planningTask: ExecutionTaskPlanningApply = { type: 'Planning', @@ -347,11 +353,7 @@ export class PageTaskExecutor { executor: async (param) => { const pageContext = await this.insight.contextRetrieverFn(); let planResult: { plans: PlanningAction[] }; - const planCache = this.taskCache.readCache( - pageContext, - 'plan', - userPrompt, - ); + const planCache = cacheGroup.readCache(pageContext, 'plan', userPrompt); if (planCache) { planResult = planCache; } else { @@ -364,7 +366,7 @@ export class PageTaskExecutor { // eslint-disable-next-line prefer-destructuring plans = planResult.plans; - this.taskCache.saveCache({ + cacheGroup.saveCache({ type: 'plan', pageContext: { url: pageContext.url, @@ -393,7 +395,7 @@ export class PageTaskExecutor { } // append tasks - const executables = await this.convertPlanToExecutable(plans); + const executables = await this.convertPlanToExecutable(plans, cacheGroup); await taskExecutor.append(executables); // flush actions diff --git a/packages/web-integration/src/common/utils.ts b/packages/web-integration/src/common/utils.ts index a264c1be..6f07b777 100644 --- a/packages/web-integration/src/common/utils.ts +++ b/packages/web-integration/src/common/utils.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import type { ElementInfo } from '@/extractor'; import type { PlaywrightParserOpt, UIContext } from '@midscene/core'; import { getTmpFile } from '@midscene/core/utils'; +import { findNearestPackageJson } from '@midscene/shared/fs'; import { base64Encoded, imageInfoOfBase64 } from '@midscene/shared/img'; import dayjs from 'dayjs'; import { WebElementInfo } from '../web-element'; @@ -81,28 +82,6 @@ async function alignElements( return textsAligned; } -/** - * Find the nearest package.json file recursively - * @param {string} dir - Home directory - * @returns {string|null} - The most recent package.json file path or null - */ -export function findNearestPackageJson(dir: string): string | null { - const packageJsonPath = path.join(dir, 'package.json'); - - if (fs.existsSync(packageJsonPath)) { - return dir; - } - - const parentDir = path.dirname(dir); - - // Return null if the root directory has been reached - if (parentDir === dir) { - return null; - } - - return findNearestPackageJson(parentDir); -} - export function reportFileName(tag = 'web') { const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss-SSS'); return `${tag}-${dateTimeInFileName}`; @@ -111,3 +90,73 @@ export function reportFileName(tag = 'web') { export function printReportMsg(filepath: string) { console.log('Midscene - report file updated:', filepath); } + +/** + * Get the current execution file name + * @returns The name of the current execution file + */ +export function getCurrentExecutionFile(): string { + const error = new Error(); + const stackTrace = error.stack; + const pkgDir = process.cwd() || ''; + if (stackTrace) { + const stackLines = stackTrace.split('\n'); + for (const line of stackLines) { + if ( + line.includes('.spec.') || + line.includes('.test.') || + line.includes('.ts') || + line.includes('.js') + ) { + const match = line.match(/(?:at\s+)?(.*?\.(?:spec|test)\.[jt]s)/); + if (match?.[1]) { + const targetFileName = match[1] + .replace(pkgDir, '') + .trim() + .replace('at ', ''); + return targetFileName; + } + } + } + } + throw new Error( + `Can not find current execution file: \n __dirname: ${__dirname}, \n stackTrace: ${stackTrace}`, + ); +} + +/** + * Generates a unique cache ID based on the current execution file and a counter. + * + * This function creates a cache ID by combining the name of the current execution file + * (typically a test or spec file) with an incrementing index. This ensures that each + * cache ID is unique within the context of a specific test file across multiple executions. + * + * The function uses a Map to keep track of the index for each unique file, incrementing + * it with each call for the same file. + * + * @returns {string} A unique cache ID in the format "filename-index" + * + * @example + * // First call for "example.spec.ts" + * generateCacheId(); // Returns "example.spec.ts-1" + * + * // Second call for "example.spec.ts" + * generateCacheId(); // Returns "example.spec.ts-2" + * + * // First call for "another.test.ts" + * generateCacheId(); // Returns "another.test.ts-1" + */ + +const testFileIndex = new Map(); +export function generateCacheId(fileName?: string): string { + const taskFile = fileName || getCurrentExecutionFile(); + if (testFileIndex.has(taskFile)) { + const currentIndex = testFileIndex.get(taskFile); + if (currentIndex !== undefined) { + testFileIndex.set(taskFile, currentIndex + 1); + } + } else { + testFileIndex.set(taskFile, 1); + } + return `${taskFile}-${testFileIndex.get(taskFile)}`; +} diff --git a/packages/web-integration/src/playwright/ai-fixture.ts b/packages/web-integration/src/playwright/ai-fixture.ts index 73ea15d0..bb902cc5 100644 --- a/packages/web-integration/src/playwright/ai-fixture.ts +++ b/packages/web-integration/src/playwright/ai-fixture.ts @@ -5,7 +5,6 @@ import type { AgentWaitForOpt } from '@midscene/core/.'; import { type TestInfo, type TestType, test } from '@playwright/test'; import type { Page as OriginPlaywrightPage } from 'playwright'; import type { PageTaskExecutor } from '../common/tasks'; -import { readTestCache, writeTestCache } from './cache'; export type APITestType = Pick, 'step'>; @@ -41,15 +40,12 @@ export const PlaywrightAiFixture = () => { (page as any)[midsceneAgentKeyId] = idForPage; const { testId } = testInfo; const { taskFile, taskTitle } = groupAndCaseForTest(testInfo); - const testCase = readTestCache(taskFile, taskTitle) || { - aiTasks: [], - }; pageAgentMap[idForPage] = new PageAgent(new PlaywrightPage(page), { testId: `playwright-${testId}-${idForPage}`, + cacheId: taskFile, groupName: taskTitle, groupDescription: taskFile, - cache: testCase, generateReport: false, // we will generate it in the reporter }); } @@ -76,7 +72,6 @@ export const PlaywrightAiFixture = () => { use: any, testInfo: TestInfo, ) => { - const { taskFile, taskTitle } = groupAndCaseForTest(testInfo); const agent = agentForPage(page, testInfo); await use( async (taskPrompt: string, opts?: { type?: 'action' | 'query' }) => { @@ -90,8 +85,6 @@ export const PlaywrightAiFixture = () => { }); }, ); - const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache(); - writeTestCache(taskFile, taskTitle, taskCacheJson); updateDumpAnnotation(testInfo, agent.dumpDataString()); }, aiAction: async ( @@ -99,7 +92,6 @@ export const PlaywrightAiFixture = () => { use: any, testInfo: TestInfo, ) => { - const { taskFile, taskTitle } = groupAndCaseForTest(testInfo); const agent = agentForPage(page, testInfo); await use(async (taskPrompt: string) => { test.step(`aiAction - ${taskPrompt}`, async () => { @@ -107,7 +99,6 @@ export const PlaywrightAiFixture = () => { await agent.aiAction(taskPrompt); }); }); - // Why there's no cache here ? updateDumpAnnotation(testInfo, agent.dumpDataString()); }, aiQuery: async ( diff --git a/packages/web-integration/src/playwright/cache.ts b/packages/web-integration/src/playwright/cache.ts deleted file mode 100644 index db94602a..00000000 --- a/packages/web-integration/src/playwright/cache.ts +++ /dev/null @@ -1,76 +0,0 @@ -import fs from 'node:fs'; -import path, { dirname, join } from 'node:path'; -import type { AiTaskCache } from '@/common/task-cache'; -import { findNearestPackageJson } from '@/common/utils'; -import { - getLogDirByType, - stringifyDumpData, - writeLogFile, -} from '@midscene/core/utils'; - -export function writeTestCache( - taskFile: string, - taskTitle: string, - taskCacheJson: AiTaskCache, -) { - const packageJson = getPkgInfo(); - writeLogFile({ - fileName: `${taskFile}(${taskTitle})`, - fileExt: 'json', - fileContent: stringifyDumpData( - { - pkgName: packageJson.name, - pkgVersion: packageJson.version, - taskFile, - taskTitle, - ...taskCacheJson, - }, - 2, - ), - type: 'cache', - }); -} - -export function readTestCache(taskFile: string, taskTitle: string) { - const cacheFile = join( - getLogDirByType('cache'), - `${taskFile}(${taskTitle}).json`, - ); - const pkgInfo = getPkgInfo(); - if (process.env.MIDSCENE_CACHE === 'true' && fs.existsSync(cacheFile)) { - try { - const data = fs.readFileSync(cacheFile, 'utf8'); - const jsonData = JSON.parse(data); - if ( - jsonData.pkgName !== pkgInfo.name || - jsonData.pkgVersion !== pkgInfo.version - ) { - return undefined; - } - return jsonData as AiTaskCache; - } catch (err) { - return undefined; - } - } - return undefined; -} - -function getPkgInfo(): { name: string; version: string } { - const packageJsonDir = findNearestPackageJson(__dirname); - if (!packageJsonDir) { - console.error('Cannot find package.json directory: ', __dirname); - return { - name: '@midscene/web', - version: '0.0.0', - }; - } - - const packageJsonPath = path.join(packageJsonDir, 'package.json'); - const data = fs.readFileSync(packageJsonPath, 'utf8'); - const packageJson = JSON.parse(data); - - return { - name: packageJson.name, - version: packageJson.version, - }; -} diff --git a/packages/web-integration/src/playwright/page.ts b/packages/web-integration/src/playwright/page.ts index d2e08938..b2a8c67b 100644 --- a/packages/web-integration/src/playwright/page.ts +++ b/packages/web-integration/src/playwright/page.ts @@ -26,7 +26,6 @@ export class Page implements AbstractPage { return this.browser.screenshot({ path, type: 'png', - quality: 75, }); } diff --git a/packages/web-integration/tests/ai/appium/ios.test.ts b/packages/web-integration/tests/ai/appium/ios.test.ts deleted file mode 100644 index e0a89831..00000000 --- a/packages/web-integration/tests/ai/appium/ios.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { AppiumAgent } from '@/appium'; -import { describe, it, vi } from 'vitest'; -import { launchPage } from './utils'; - -vi.setConfig({ - testTimeout: 90 * 1000, -}); - -const IOS_DEFAULT_OPTIONS = { - port: 4723, - capabilities: { - platformName: 'iOS', - 'appium:automationName': 'XCUITest', - 'appium:deviceName': 'iPhone 15 Plus Simulator (18.0)', - 'appium:platformVersion': '18.0', - 'appium:bundleId': 'com.apple.Preferences', - 'appium:udid': 'B8517A53-6C4C-41D8-9B1E-825A0D75FA47', - 'appium:newCommandTimeout': 600, - }, -}; - -describe( - 'appium integration', - async () => { - await it('iOS settings page demo for input', async () => { - const page = await launchPage(IOS_DEFAULT_OPTIONS); - const mid = new AppiumAgent(page); - - await mid.aiAction('输入框中输入“123”'); - await mid.aiAction('输入框中输入“456”'); - await mid.aiAction('输入框中输入“789”'); - }); - await it('iOS settings page demo for scroll', async () => { - const page = await launchPage(IOS_DEFAULT_OPTIONS); - const mid = new AppiumAgent(page); - - await mid.aiAction('滑动列表到底部'); - await mid.aiAction('打开"开发者"'); - await mid.aiAction('滑动列表到底部'); - await mid.aiAction('滑动列表到顶部'); - await mid.aiAction('向下滑动一屏'); - await mid.aiAction('向上滑动一屏'); - }); - }, - { - timeout: 360 * 1000, - }, -); diff --git a/packages/web-integration/tests/ai/appium/android.test.ts b/packages/web-integration/tests/ai/native/appium/android.test.ts similarity index 100% rename from packages/web-integration/tests/ai/appium/android.test.ts rename to packages/web-integration/tests/ai/native/appium/android.test.ts diff --git a/packages/web-integration/tests/ai/appium/dongchedi.test.ts b/packages/web-integration/tests/ai/native/appium/dongchedi.test.ts similarity index 73% rename from packages/web-integration/tests/ai/appium/dongchedi.test.ts rename to packages/web-integration/tests/ai/native/appium/dongchedi.test.ts index e8449c6c..842b38ef 100644 --- a/packages/web-integration/tests/ai/appium/dongchedi.test.ts +++ b/packages/web-integration/tests/ai/native/appium/dongchedi.test.ts @@ -1,3 +1,4 @@ +import path from 'node:path'; import { AppiumAgent } from '@/appium'; import { describe, expect, it, vi } from 'vitest'; import { launchPage } from './utils'; @@ -13,12 +14,13 @@ const IOS_OPTIONS = { capabilities: { platformName: 'iOS', 'appium:automationName': 'XCUITest', - 'appium:deviceName': 'iPhone 15 Plus Simulator (18.0)', - 'appium:platformVersion': '18.0', + 'appium:deviceName': 'iPhone 15 Pro Simulator (17.5)', + 'appium:platformVersion': '17.5', 'appium:bundleId': 'com.ss.ios.InHouse.AutoMobile', - 'appium:udid': 'B8517A53-6C4C-41D8-9B1E-825A0D75FA47', + 'appium:udid': '9ADCE031-36DF-4025-8C62-073FC7FAB901', 'appium:newCommandTimeout': 600, }, + outputDir: path.join(__dirname, 'tmp'), }; const ANDROID_OPTIONS = { @@ -37,15 +39,13 @@ describe( 'appium integration', () => { it('懂车帝查找小米 SU7', async () => { - const page = await launchPage(ANDROID_OPTIONS); + const page = await launchPage(IOS_OPTIONS); const mid = new AppiumAgent(page); - await mid.aiAction('点击同意按钮'); - await sleep(3000); - await mid.aiAction('点击顶部搜索框'); - await sleep(3000); - await mid.aiAction('在搜索框里输入"SU7"，并点击搜索'); - await sleep(3000); + await mid.aiAction('点击顶部输入框'); + // await generateExtractData(page, './tmp'); + await mid.aiAction('在输入框里输入"小米SU7"，并点击搜索'); + // await sleep(3000); const items = await mid.aiQuery( '"{carName: string, price: number }[], return item name, price', ); diff --git a/packages/web-integration/tests/ai/native/appium/ios.test.ts b/packages/web-integration/tests/ai/native/appium/ios.test.ts new file mode 100644 index 00000000..c10d073f --- /dev/null +++ b/packages/web-integration/tests/ai/native/appium/ios.test.ts @@ -0,0 +1,51 @@ +import { AppiumAgent } from '@/appium'; +import { describe, it, vi } from 'vitest'; +import { launchPage } from './utils'; + +vi.setConfig({ + testTimeout: 90 * 1000, +}); + +const IOS_DEFAULT_OPTIONS = { + port: 4723, + capabilities: { + platformName: 'iOS', + 'appium:automationName': 'XCUITest', + 'appium:deviceName': 'iPhone 15 Pro Simulator (17.5)', + 'appium:platformVersion': '17.5', + // 'appium:bundleId': 'com.apple.Preferences', + 'appium:bundleId': 'com.ss.iphone.ugc.AwemeInhouse', + 'appium:udid': '9ADCE031-36DF-4025-8C62-073FC7FAB901', + 'appium:newCommandTimeout': 600, + }, +}; + +describe( + 'appium integration', + async () => { + await it('iOS settings page demo for input', async () => { + const page = await launchPage(IOS_DEFAULT_OPTIONS); + const mid = new AppiumAgent(page); + + await mid.aiAction('点击同意按钮'); + await mid.aiAction('点击底部朋友'); + // await mid.aiAction('输入框中输入“123”'); + // await mid.aiAction('输入框中输入“456”'); + // await mid.aiAction('输入框中输入“789”'); + }); + // await it('iOS settings page demo for scroll', async () => { + // const page = await launchPage(IOS_DEFAULT_OPTIONS); + // const mid = new AppiumAgent(page); + + // await mid.aiAction('滑动列表到底部'); + // await mid.aiAction('打开"开发者"'); + // await mid.aiAction('滑动列表到底部'); + // await mid.aiAction('滑动列表到顶部'); + // await mid.aiAction('向下滑动一屏'); + // await mid.aiAction('向上滑动一屏'); + // }); + }, + { + timeout: 360 * 1000, + }, +); diff --git a/packages/web-integration/tests/ai/appium/utils.ts b/packages/web-integration/tests/ai/native/appium/utils.ts similarity index 100% rename from packages/web-integration/tests/ai/appium/utils.ts rename to packages/web-integration/tests/ai/native/appium/utils.ts diff --git a/packages/web-integration/tests/ai/playright-report/todo-report.spec.ts b/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts similarity index 83% rename from packages/web-integration/tests/ai/playright-report/todo-report.spec.ts rename to packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts index f072fc45..89cad9a7 100644 --- a/packages/web-integration/tests/ai/playright-report/todo-report.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts @@ -1,13 +1,13 @@ import path from 'node:path'; import { expect } from 'playwright/test'; -import { test } from '../playright/fixture'; -import { getLastModifiedReportHTMLFile } from '../playright/util'; +import { test } from '../playwright/fixture'; +import { getLastModifiedReportHTMLFile } from '../playwright/util'; test('ai report', async ({ page, ai, aiAssert }, testInfo) => { testInfo.snapshotSuffix = ''; await new Promise((resolve) => setTimeout(resolve, 3000)); const htmlFile = getLastModifiedReportHTMLFile( - path.join(__dirname, '../../../midscene_run/report'), + path.join(__dirname, '../../../../midscene_run/report'), ); console.log('report html path:', htmlFile); await page.goto(`file:${htmlFile}`); diff --git a/packages/web-integration/tests/ai/playright-report/todo-report.spec.ts-snapshots/ai-report-1-report.txt b/packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt similarity index 100% rename from packages/web-integration/tests/ai/playright-report/todo-report.spec.ts-snapshots/ai-report-1-report.txt rename to packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt diff --git a/packages/web-integration/tests/ai/playright/ai-auto-todo.spec.ts b/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts similarity index 93% rename from packages/web-integration/tests/ai/playright/ai-auto-todo.spec.ts rename to packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts index 15ff820b..9675839d 100644 --- a/packages/web-integration/tests/ai/playright/ai-auto-todo.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts @@ -5,8 +5,7 @@ test.beforeEach(async ({ page }) => { await page.goto('https://todomvc.com/examples/react/dist/'); }); -const CACHE_TIME_OUT = - Boolean(process.env.MIDSCENE_CACHE) && Boolean(process.env.GITHUB_ACTIONS); +const CACHE_TIME_OUT = process.env.MIDSCENE_CACHE; test('ai todo', async ({ ai, aiQuery }) => { if (CACHE_TIME_OUT) { diff --git a/packages/web-integration/tests/ai/playright/ai-online-order.spec.ts b/packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts similarity index 100% rename from packages/web-integration/tests/ai/playright/ai-online-order.spec.ts rename to packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts index bc5f74aa..7402f2cb 100644 --- a/packages/web-integration/tests/ai/playright/ai-online-order.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts @@ -33,12 +33,12 @@ test('ai online order', async ({ ai, page, aiQuery }) => { // productPrice: "商品价格", // productDescription: "商品描述（饮品的各种参数，吸管、冰沙等），在价格下面", // })); - expect(cardDetail.productName).toContain('多肉葡萄'); - expect(cardDetail.productDescription).toContain('绿妍'); console.log('商品订单详情：', { productName: cardDetail.productName, productPrice: cardDetail.productPrice, productDescription: cardDetail.productDescription, }); + expect(cardDetail.productName).toContain('多肉葡萄'); + expect(cardDetail.productDescription).toContain('绿妍'); }); diff --git a/packages/web-integration/tests/ai/playright/fixture.ts b/packages/web-integration/tests/ai/web/playwright/fixture.ts similarity index 100% rename from packages/web-integration/tests/ai/playright/fixture.ts rename to packages/web-integration/tests/ai/web/playwright/fixture.ts diff --git a/packages/web-integration/tests/ai/playright/generate-test-data.spec.ts b/packages/web-integration/tests/ai/web/playwright/generate-test-data.spec.ts similarity index 100% rename from packages/web-integration/tests/ai/playright/generate-test-data.spec.ts rename to packages/web-integration/tests/ai/web/playwright/generate-test-data.spec.ts diff --git a/packages/web-integration/tests/ai/playright/todo-app-midscene.spec.ts b/packages/web-integration/tests/ai/web/playwright/todo-app-midscene.spec.ts similarity index 100% rename from packages/web-integration/tests/ai/playright/todo-app-midscene.spec.ts rename to packages/web-integration/tests/ai/web/playwright/todo-app-midscene.spec.ts diff --git a/packages/web-integration/tests/ai/playright/tool.ts b/packages/web-integration/tests/ai/web/playwright/tool.ts similarity index 100% rename from packages/web-integration/tests/ai/playright/tool.ts rename to packages/web-integration/tests/ai/web/playwright/tool.ts diff --git a/packages/web-integration/tests/ai/playright/util.ts b/packages/web-integration/tests/ai/web/playwright/util.ts similarity index 95% rename from packages/web-integration/tests/ai/playright/util.ts rename to packages/web-integration/tests/ai/web/playwright/util.ts index f1b02f61..a3ee3685 100644 --- a/packages/web-integration/tests/ai/playright/util.ts +++ b/packages/web-integration/tests/ai/web/playwright/util.ts @@ -30,7 +30,7 @@ export function getLastModifiedReportHTMLFile(dirPath: string) { if (stats.mtimeMs > latestMtime) { latestMtime = stats.mtimeMs; latestFile = filePath; - console.log('filePath', filePath); + // console.log('filePath', filePath); } } } diff --git a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts b/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts similarity index 97% rename from packages/web-integration/tests/ai/puppeteer/showcase.test.ts rename to packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts index 4cc533dd..0e989458 100644 --- a/packages/web-integration/tests/ai/puppeteer/showcase.test.ts +++ b/packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts @@ -2,10 +2,6 @@ import { PuppeteerAgent } from '@/puppeteer'; import { describe, expect, it, vi } from 'vitest'; import { launchPage } from './utils'; -vi.setConfig({ - testTimeout: 140 * 1000, -}); - describe( 'puppeteer integration', () => { diff --git a/packages/web-integration/tests/ai/puppeteer/utils.ts b/packages/web-integration/tests/ai/web/puppeteer/utils.ts similarity index 100% rename from packages/web-integration/tests/ai/puppeteer/utils.ts rename to packages/web-integration/tests/ai/web/puppeteer/utils.ts diff --git a/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/input.png b/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/input.png new file mode 100644 index 00000000..d63748e0 Binary files /dev/null and b/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/input.png differ diff --git a/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/output.png b/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/output.png new file mode 100644 index 00000000..ffb6077b Binary files /dev/null and b/packages/web-integration/tests/unit-test/fixtures/extractor/scroll/output.png differ diff --git a/packages/web-integration/tests/unit-test/task-cache.test.ts b/packages/web-integration/tests/unit-test/task-cache.test.ts index f7030054..b0ec5920 100644 --- a/packages/web-integration/tests/unit-test/task-cache.test.ts +++ b/packages/web-integration/tests/unit-test/task-cache.test.ts @@ -23,7 +23,8 @@ describe('TaskCache', () => { }); it('should return false if no cache is available', async () => { - const result = taskCache.readCache( + const cacheGroup = taskCache.getCacheGroupByPrompt('test prompt'); + const result = cacheGroup.readCache( formalPageContext, 'plan', 'test prompt', @@ -32,17 +33,21 @@ describe('TaskCache', () => { }); it('should return false if the prompt does not match', async () => { - taskCache.cache = { - aiTasks: [ - { - type: 'plan', - prompt: 'different prompt', - pageContext, - response: { plans: [] }, - }, - ], - }; - const result = taskCache.readCache( + taskCache.cache.aiTasks = [ + { + prompt: 'different prompt', + tasks: [ + { + type: 'plan', + prompt: 'different prompt', + pageContext, + response: { plans: [] }, + }, + ], + }, + ]; + const cacheGroup = taskCache.getCacheGroupByPrompt('test prompt'); + const result = cacheGroup.readCache( formalPageContext, 'plan', 'test prompt', @@ -54,16 +59,22 @@ describe('TaskCache', () => { taskCache.cache = { aiTasks: [ { - type: 'locate', prompt: 'test prompt', - pageContext, - response: { - elements: [{ id: 'element3' }], - } as AIElementParseResponse, + tasks: [ + { + type: 'locate', + prompt: 'test prompt', + pageContext, + response: { + elements: [{ id: 'element3' }], + } as AIElementParseResponse, + }, + ], }, ], }; - const result = taskCache.readCache( + const cacheGroup = taskCache.getCacheGroupByPrompt('test prompt'); + const result = cacheGroup.readCache( formalPageContext, 'locate', 'test prompt', @@ -78,14 +89,21 @@ describe('TaskCache', () => { taskCache.cache = { aiTasks: [ { - type: 'plan', prompt: 'test prompt', - pageContext, - response: cachedResponse, + tasks: [ + { + type: 'plan', + prompt: 'test prompt', + pageContext, + response: cachedResponse, + }, + ], }, ], }; - const result = taskCache.readCache( + + const cacheGroup = taskCache.getCacheGroupByPrompt('test prompt'); + const result = cacheGroup.readCache( formalPageContext, 'plan', 'test prompt', @@ -94,14 +112,15 @@ describe('TaskCache', () => { }); it('should save cache correctly', () => { + const cacheGroup = taskCache.getCacheGroupByPrompt('test prompt'); const newCache: PlanTask = { type: 'plan', prompt: 'new prompt', pageContext, response: { plans: [{ type: 'Locate', thought: '', param: {} }] }, }; - taskCache.saveCache(newCache); - expect(taskCache.newCache.aiTasks).toContain(newCache); + cacheGroup.saveCache(newCache); + expect(taskCache.newCache.aiTasks[0].tasks).toContain(newCache); }); it('should check page context equality correctly', () => { diff --git a/packages/web-integration/tests/unit-test/util.test.ts b/packages/web-integration/tests/unit-test/util.test.ts new file mode 100644 index 00000000..8840e202 --- /dev/null +++ b/packages/web-integration/tests/unit-test/util.test.ts @@ -0,0 +1,9 @@ +import { getCurrentExecutionFile } from '@/common/utils'; +import { beforeEach, describe, expect, it } from 'vitest'; + +describe('TaskCache', () => { + it('should return the current execution file', () => { + const currentExecutionFile = getCurrentExecutionFile(); + expect(currentExecutionFile).toBe('/tests/unit-test/util.test.ts'); + }); +}); diff --git a/packages/web-integration/tests/unit-test/web-extractor.test.ts b/packages/web-integration/tests/unit-test/web-extractor.test.ts index d161d78f..53d6c903 100644 --- a/packages/web-integration/tests/unit-test/web-extractor.test.ts +++ b/packages/web-integration/tests/unit-test/web-extractor.test.ts @@ -2,7 +2,7 @@ import path, { join } from 'node:path'; import { parseContextFromWebPage } from '@/common/utils'; import { generateExtractData } from '@/debug'; import { describe, expect, it } from 'vitest'; -import { launchPage } from '../ai/puppeteer/utils'; +import { launchPage } from '../ai/web/puppeteer/utils'; const pagePath = join(__dirname, './fixtures/web-extractor/index.html'); describe( diff --git a/packages/web-integration/vitest.config.ts b/packages/web-integration/vitest.config.ts index 52a9e7cf..0f5e51d4 100644 --- a/packages/web-integration/vitest.config.ts +++ b/packages/web-integration/vitest.config.ts @@ -11,9 +11,21 @@ dotenv.config({ path: path.join(__dirname, '../../.env'), }); -const enableAiTest = Boolean(process.env.AITEST); +const aiTestType = process.env.AI_TEST_TYPE; const unitTests = ['tests/unit-test/*.test.ts']; -const aiTests = ['tests/ai/**/*.test.ts']; +const aiWebTests = ['tests/ai/web/**/*.test.ts']; +const aiNativeTests = ['tests/ai/native/**/*.test.ts']; +// const aiNativeTests = ['tests/ai/native/appium/dongchedi.test.ts']; +const testFiles = (() => { + switch (aiTestType) { + case 'web': + return [...unitTests, ...aiWebTests]; + case 'native': + return [...aiNativeTests]; + default: + return unitTests; + } +})(); export default defineConfig({ resolve: { @@ -22,6 +34,6 @@ export default defineConfig({ }, }, test: { - include: enableAiTest ? [...aiTests, ...unitTests] : unitTests, + include: testFiles, }, });