diff --git a/packages/web-integration/src/appium/page.ts b/packages/web-integration/src/appium/page.ts index 16aaa24a..cde588db 100644 --- a/packages/web-integration/src/appium/page.ts +++ b/packages/web-integration/src/appium/page.ts @@ -3,7 +3,7 @@ import { resizeImg } from '@midscene/shared/img'; import { DOMParser } from '@xmldom/xmldom'; import type { KeyInput as PuppeteerKeyInput } from 'puppeteer'; import type { Browser } from 'webdriverio'; -import { clientExtractTextWithPosition, type ElementInfo } from '../extractor'; +import { type ElementInfo, clientExtractTextWithPosition } from '../extractor'; import type { AbstractPage, MouseButton, screenshotOptions } from '../page'; type WebKeyInput = PuppeteerKeyInput; @@ -138,6 +138,10 @@ export class Page implements AbstractPage { actions.push({ type: 'keyUp', value: char }); } + if (!actions.length) { + return; + } + await this.browser.performActions([ { type: 'key', diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index 11f01407..33ed16b1 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -27,10 +27,10 @@ import Insight, { import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils'; import { base64Encoded } from '@midscene/shared/img'; import type { KeyInput } from 'puppeteer'; +import type { ElementInfo } from '../extractor'; import type { WebElementInfo } from '../web-element'; import { type AiTaskCache, TaskCache } from './task-cache'; import { type WebUIContext, parseContextFromWebPage } from './utils'; -import type { ElementInfo } from '../extractor'; interface ExecutionResult { output: OutputType; @@ -208,10 +208,16 @@ export class PageTaskExecutor { param: plan.param, executor: async (taskParam, { element }) => { if (element) { - await this.page.clearInput(element as ElementInfo); + if (taskParam.value === '') { + await this.page.clearInput(element as ElementInfo); + } else { + await this.page.mouse.click( + element.center[0], + element.center[1], + ); + await this.page.keyboard.type(taskParam.value); + } } - - await this.page.keyboard.type(taskParam.value); }, }; return taskActionInput; diff --git a/packages/web-integration/src/extractor/client-extractor.ts b/packages/web-integration/src/extractor/client-extractor.ts index 16afef60..9103b6f3 100644 --- a/packages/web-integration/src/extractor/client-extractor.ts +++ b/packages/web-integration/src/extractor/client-extractor.ts @@ -69,10 +69,14 @@ function getXPathForElement(element: Node): string { return count; }; - const buildAttributePart = (elem: Element, attributes: string[]): string => { + const buildAttributePart = (elem: Element): string => { + const attributes = ['id', 'resource-id', 'content-desc', 'text', 'class']; for (const attr of attributes) { if (elem.hasAttribute(attr)) { - return `[@${attr}="${elem.getAttribute(attr)}"]`; + const value = elem.getAttribute(attr); + if (value && value.trim() !== '') { + return `[@${attr}="${value}"]`; + } } } return ''; @@ -86,15 +90,20 @@ function getXPathForElement(element: Node): string { if (node.nodeType === 1) { const elem = node as Element; const tagName = elem.nodeName.toLowerCase(); - const index = getIndex(node, node.nodeName); - let part = `/${tagName}${index > 1 ? `[${index}]` : ''}`; - - const attributes = ['id', 'resource-id', 'content-desc', 'text', 'class']; - - const attributePart = buildAttributePart(elem, attributes); - - // 如果找到有意义的属性，则替代掉index部分 - part = attributePart ? `/${tagName}${attributePart}` : part; + let part = `/${tagName}`; + + const attributePart = buildAttributePart(elem); + + // 如果找到有意义的属性，则添加属性部分 + if (attributePart) { + part += attributePart; + } else { + // 如果没有有意义的属性，则添加索引 + const index = getIndex(node, node.nodeName); + if (index > 1) { + part += `[${index}]`; + } + } path += part; } @@ -120,9 +129,7 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] { parentNode.children.push(currentNodeDes); } - if (node.childNodes && node.childNodes.length === 0) { - collectElementInfo(node); - } + collectElementInfo(node); if (node.childNodes && node.childNodes.length > 0) { for (let i = 0; i < node.childNodes.length; i++) { @@ -149,8 +156,25 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] { case 'BUTTON': nodeType = NodeType.BUTTON; break; - default: + case 'SEARCHINPUT': + case 'INPUT': + nodeType = NodeType.FORM_ITEM; + break; + case 'NAV': + case 'LIST': + case 'CELL': nodeType = NodeType.CONTAINER; + break; + default: + if ( + attributes.id === 'android:id/input' || + attributes.id === 'android:id/inputArea' + ) { + nodeType = NodeType.FORM_ITEM; + } else { + nodeType = NodeType.CONTAINER; + } + break; } const xpath = getXPathForElement(node); @@ -174,7 +198,9 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] { nodePath: '', }; - elementInfoArray.push(elementInfo); + if (elementInfo.nodeType !== NodeType.CONTAINER) { + elementInfoArray.push(elementInfo); + } } const rootNode = initNode; diff --git a/packages/web-integration/src/extractor/web-extractor.ts b/packages/web-integration/src/extractor/web-extractor.ts index 418850ee..65e79801 100644 --- a/packages/web-integration/src/extractor/web-extractor.ts +++ b/packages/web-integration/src/extractor/web-extractor.ts @@ -254,4 +254,3 @@ export function extractTextWithPosition( } return elementInfoArray; } - diff --git a/packages/web-integration/tests/unit-test/fixtures/client-extractor/android-setting.xml b/packages/web-integration/tests/unit-test/fixtures/client-extractor/android-setting.xml index bd67afe7..3d724cc3 100644 --- a/packages/web-integration/tests/unit-test/fixtures/client-extractor/android-setting.xml +++ b/packages/web-integration/tests/unit-test/fixtures/client-extractor/android-setting.xml @@ -7,136 +7,133 @@ - - - - - - + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + - - - + + + - - - + + - - - - - + + - - - - - + - - - + + + - - - + + + - + - - - + + + + + + - - - + + - + - - - + + + - - - + + - + + + + - - - + + + - - - + + + - + - - - + + + - - - + + + - + - - - + + + - - - + + + - + - - - + + + - - - + + + - + + + + + + - @@ -146,4 +143,4 @@ - +'