fix: distinct clear input and input

web-infra-dev · Sep 4, 2024 · a806b24 · a806b24
1 parent adbbdfd
commit a806b24
Show file tree

Hide file tree

Showing 5 changed files with 152 additions and 120 deletions.
diff --git a/packages/web-integration/src/appium/page.ts b/packages/web-integration/src/appium/page.ts
@@ -3,7 +3,7 @@ import { resizeImg } from '@midscene/shared/img';
 import { DOMParser } from '@xmldom/xmldom';
 import type { KeyInput as PuppeteerKeyInput } from 'puppeteer';
 import type { Browser } from 'webdriverio';
-import { clientExtractTextWithPosition, type ElementInfo } from '../extractor';
+import { type ElementInfo, clientExtractTextWithPosition } from '../extractor';
 import type { AbstractPage, MouseButton, screenshotOptions } from '../page';
 
 type WebKeyInput = PuppeteerKeyInput;
@@ -138,6 +138,10 @@ export class Page implements AbstractPage {
       actions.push({ type: 'keyUp', value: char });
     }
 
+    if (!actions.length) {
+      return;
+    }
+
     await this.browser.performActions([
       {
         type: 'key',

diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts
@@ -27,10 +27,10 @@ import Insight, {
 import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils';
 import { base64Encoded } from '@midscene/shared/img';
 import type { KeyInput } from 'puppeteer';
+import type { ElementInfo } from '../extractor';
 import type { WebElementInfo } from '../web-element';
 import { type AiTaskCache, TaskCache } from './task-cache';
 import { type WebUIContext, parseContextFromWebPage } from './utils';
-import type { ElementInfo } from '../extractor';
 
 interface ExecutionResult<OutputType = any> {
   output: OutputType;
@@ -208,10 +208,16 @@ export class PageTaskExecutor {
               param: plan.param,
               executor: async (taskParam, { element }) => {
                 if (element) {
-                  await this.page.clearInput(element as ElementInfo);
+                  if (taskParam.value === '') {
+                    await this.page.clearInput(element as ElementInfo);
+                  } else {
+                    await this.page.mouse.click(
+                      element.center[0],
+                      element.center[1],
+                    );
+                    await this.page.keyboard.type(taskParam.value);
+                  }
                 }
-
-                await this.page.keyboard.type(taskParam.value);
               },
             };
           return taskActionInput;

diff --git a/packages/web-integration/src/extractor/client-extractor.ts b/packages/web-integration/src/extractor/client-extractor.ts
@@ -69,10 +69,14 @@ function getXPathForElement(element: Node): string {
     return count;
   };
 
-  const buildAttributePart = (elem: Element, attributes: string[]): string => {
+  const buildAttributePart = (elem: Element): string => {
+    const attributes = ['id', 'resource-id', 'content-desc', 'text', 'class'];
     for (const attr of attributes) {
       if (elem.hasAttribute(attr)) {
-        return `[@${attr}="${elem.getAttribute(attr)}"]`;
+        const value = elem.getAttribute(attr);
+        if (value && value.trim() !== '') {
+          return `[@${attr}="${value}"]`;
+        }
       }
     }
     return '';
@@ -86,15 +90,20 @@ function getXPathForElement(element: Node): string {
     if (node.nodeType === 1) {
       const elem = node as Element;
       const tagName = elem.nodeName.toLowerCase();
-      const index = getIndex(node, node.nodeName);
-      let part = `/${tagName}${index > 1 ? `[${index}]` : ''}`;
-
-      const attributes = ['id', 'resource-id', 'content-desc', 'text', 'class'];
-
-      const attributePart = buildAttributePart(elem, attributes);
-
-      // 如果找到有意义的属性，则替代掉index部分
-      part = attributePart ? `/${tagName}${attributePart}` : part;
+      let part = `/${tagName}`;
+
+      const attributePart = buildAttributePart(elem);
+
+      // 如果找到有意义的属性，则添加属性部分
+      if (attributePart) {
+        part += attributePart;
+      } else {
+        // 如果没有有意义的属性，则添加索引
+        const index = getIndex(node, node.nodeName);
+        if (index > 1) {
+          part += `[${index}]`;
+        }
+      }
 
       path += part;
     }
@@ -120,9 +129,7 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] {
       parentNode.children.push(currentNodeDes);
     }
 
-    if (node.childNodes && node.childNodes.length === 0) {
-      collectElementInfo(node);
-    }
+    collectElementInfo(node);
 
     if (node.childNodes && node.childNodes.length > 0) {
       for (let i = 0; i < node.childNodes.length; i++) {
@@ -149,8 +156,25 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] {
       case 'BUTTON':
         nodeType = NodeType.BUTTON;
         break;
-      default:
+      case 'SEARCHINPUT':
+      case 'INPUT':
+        nodeType = NodeType.FORM_ITEM;
+        break;
+      case 'NAV':
+      case 'LIST':
+      case 'CELL':
         nodeType = NodeType.CONTAINER;
+        break;
+      default:
+        if (
+          attributes.id === 'android:id/input' ||
+          attributes.id === 'android:id/inputArea'
+        ) {
+          nodeType = NodeType.FORM_ITEM;
+        } else {
+          nodeType = NodeType.CONTAINER;
+        }
+        break;
     }
 
     const xpath = getXPathForElement(node);
@@ -174,7 +198,9 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] {
       nodePath: '',
     };
 
-    elementInfoArray.push(elementInfo);
+    if (elementInfo.nodeType !== NodeType.CONTAINER) {
+      elementInfoArray.push(elementInfo);
+    }
   }
 
   const rootNode = initNode;

diff --git a/packages/web-integration/src/extractor/web-extractor.ts b/packages/web-integration/src/extractor/web-extractor.ts
@@ -254,4 +254,3 @@ export function extractTextWithPosition(
   }
   return elementInfoArray;
 }
-