fix: filter unused element for appium

.vscode/settings.json

@@ -4,8 +4,5 @@
   },
   "editor.defaultFormatter": "biomejs.biome",
   "editor.formatOnSave": true,
-  "editor.formatOnSaveMode": "modifications",
-  "[plaintext]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
-  }
+  "editor.formatOnSaveMode": "modifications"
 }

packages/shared/src/img/transform.ts

@@ -46,6 +46,10 @@ export async function transformImgPathToBase64(inputPath: string) {
  */
 export async function resizeImg(
   inputData: string | Buffer,
+  newSize?: {
+    width: number;
+    height: number;
+  },
 ): Promise<string | Buffer> {
   const isBase64 = typeof inputData === 'string';
   const imageBuffer = isBase64
@@ -59,9 +63,9 @@ export async function resizeImg(
     throw Error('Undefined width or height from the input image.');
   }
 
-  const newSize = calculateNewDimensions(width, height);
+  const finalNewSize = newSize || calculateNewDimensions(width, height);
 
-  image.resize(newSize.width, newSize.height);
+  image.resize(finalNewSize.width, finalNewSize.height);
   const resizedBuffer = await image.getBufferAsync(Jimp.MIME_PNG);
 
   return isBase64 ? resizedBuffer.toString('base64') : resizedBuffer;

packages/web-integration/src/appium/page.ts

@@ -1,9 +1,10 @@
 import fs from 'node:fs';
 import { resizeImg } from '@midscene/shared/img';
+import { DOMParser } from '@xmldom/xmldom';
 import type { KeyInput as PuppeteerKeyInput } from 'puppeteer';
 import type { Browser } from 'webdriverio';
-import type { AbstractPage, MouseButton, screenshotOptions } from '../Page';
-import { parsePageSource } from '../extractor/appium-exactor';
+import { clientExtractTextWithPosition } from '../extractor';
+import type { AbstractPage, MouseButton, screenshotOptions } from '../page';
 
 type WebKeyInput = PuppeteerKeyInput;
 
@@ -21,16 +22,32 @@ export class Page implements AbstractPage {
 
   async getElementInfos() {
     const pageSource = await this.browser.getPageSource();
-    return parsePageSource(pageSource);
+    const { width, height } = await this.browser.getWindowSize();
+    const parser = new DOMParser();
+    const doc = parser.parseFromString(pageSource, 'text/xml');
+    const infos = clientExtractTextWithPosition(doc).filter(
+      (element) =>
+        element.rect.height !== height &&
+        element.rect.width !== width &&
+        element.rect.left !== 0 &&
+        element.rect.top !== 0 &&
+        element.attributes.visible === 'true',
+    );
+
+    return infos;
   }
 
   async screenshot(options: screenshotOptions): Promise<Buffer> {
     if (!options.path) {
       return Buffer.from('');
     }
 
+    const { width, height } = await this.browser.getWindowSize();
     const screenshotBuffer = await this.browser.saveScreenshot(options.path);
-    const resizedScreenshotBuffer = await resizeImg(screenshotBuffer);
+    const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, {
+      width,
+      height,
+    });
 
     if (options.path) {
       fs.writeFileSync(options.path, resizedScreenshotBuffer);

packages/web-integration/src/common/utils.ts

@@ -2,7 +2,7 @@ import assert from 'node:assert';
 import type { Buffer } from 'node:buffer';
 import fs, { readFileSync } from 'node:fs';
 import path from 'node:path';
-import type { ElementInfo } from '@/extractor/extractor';
+import type { ElementInfo } from '@/extractor';
 import type { PlaywrightParserOpt, UIContext } from '@midscene/core';
 import { getTmpFile } from '@midscene/core/utils';
 import { base64Encoded, imageInfoOfBase64 } from '@midscene/shared/img';

packages/web-integration/src/debug/img/util.ts

@@ -1,5 +1,5 @@
+import type { ElementInfo } from '@/extractor';
 import { NodeType } from '@/extractor/constants';
-import type { ElementInfo } from '@/extractor/extractor';
 import type { WebPage } from '../../common/page';
 
 export async function getElementInfos(page: WebPage) {

packages/web-integration/src/debug/index.ts

@@ -1,8 +1,8 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 import type { WebPage } from '@/common/page';
+import type { ElementInfo } from '@/extractor';
 import { NodeType } from '@/extractor/constants';
-import type { ElementInfo } from '@/extractor/extractor';
 import { getTmpFile } from '@midscene/core/utils';
 import {
   processImageElementInfo,

packages/web-integration/src/extractor/appium-exactor.ts → packages/web-integration/src/extractor/client-extractor.ts

@@ -1,6 +1,5 @@
-import { DOMParser } from '@xmldom/xmldom';
+import type { ElementInfo } from './';
 import { NodeType } from './constants';
-import type { ElementInfo } from './extractor';
 import { generateId, midsceneGenerateHash } from './util';
 
 // https://github.com/appium/appium/tree/master/packages/universal-xml-plugin
@@ -55,11 +54,36 @@ function validTextNodeContent(node: Node): string {
   return '';
 }
 
-// New parsePageSource function to extract from Appium's pageSource
-export function parsePageSource(pageSource: string) {
-  const parser = new DOMParser();
-  const doc = parser.parseFromString(pageSource, 'text/xml');
-  return extractTextWithPosition(doc);
+function getXPathForElement(element: Node): string {
+  if (element.nodeType !== 1) {
+    return '';
+  }
+
+  const getIndex = (sib: Node, name: string) => {
+    let count = 1;
+    for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) {
+      if (cur.nodeType === 1 && cur.nodeName === name) {
+        count++;
+      }
+    }
+    return count;
+  };
+
+  const getPath = (node: Node, path = ''): string => {
+    if (node.parentNode) {
+      path = getPath(node.parentNode, path);
+    }
+
+    if (node.nodeType === 1) {
+      const index = getIndex(node, node.nodeName);
+      const tagName = node.nodeName.toLowerCase();
+      path += `/${tagName}[${index}]`;
+    }
+
+    return path;
+  };
+
+  return getPath(element);
 }
 
 // Perform DFS traversal and collect element information
@@ -107,7 +131,6 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] {
     }
 
     const xpath = getXPathForElement(node);
-
     const elementInfo: ElementInfo = {
       id: nodeHashId,
       indexId: generateId(nodeIndex++),
@@ -125,6 +148,7 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] {
       ],
       nodeType,
       htmlNode: null,
+      nodePath: '',
     };
 
     elementInfoArray.push(elementInfo);
@@ -136,35 +160,3 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] {
 
   return elementInfoArray;
 }
-
-function getXPathForElement(element: Node): string {
-  if (element.nodeType !== 1) {
-    return '';
-  }
-
-  const getIndex = (sib: Node, name: string) => {
-    let count = 1;
-    for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) {
-      if (cur.nodeType === 1 && cur.nodeName === name) {
-        count++;
-      }
-    }
-    return count;
-  };
-
-  const getPath = (node: Node, path = ''): string => {
-    if (node.parentNode) {
-      path = getPath(node.parentNode, path);
-    }
-
-    if (node.nodeType === 1) {
-      const index = getIndex(node, node.nodeName);
-      const tagName = node.nodeName.toLowerCase();
-      path += `/${tagName}[${index}]`;
-    }
-
-    return path;
-  };
-
-  return getPath(element);
-}

packages/web-integration/src/extractor/debug.ts

@@ -1,6 +1,6 @@
-import { extractTextWithPosition } from '.';
+import { webExtractTextWithPosition } from '.';
 import { setExtractTextWithPositionOnWindow } from './util';
 
-console.log(extractTextWithPosition(document.body, true));
-console.log(JSON.stringify(extractTextWithPosition(document.body, false)));
+console.log(webExtractTextWithPosition(document.body, true));
+console.log(JSON.stringify(webExtractTextWithPosition(document.body, false)));
 setExtractTextWithPositionOnWindow();

packages/web-integration/src/extractor/index.ts

@@ -1 +1,21 @@
-export { extractTextWithPosition } from './extractor';
+import type { NodeType } from '@midscene/shared/constants';
+
+export interface ElementInfo {
+  id: string;
+  indexId?: string; // for debug use
+  nodePath: string;
+  nodeHashId: string;
+  locator: string;
+  attributes: {
+    nodeType: NodeType;
+    [key: string]: string;
+  };
+  nodeType: NodeType;
+  htmlNode: Node | null;
+  content: string;
+  rect: { left: number; top: number; width: number; height: number };
+  center: [number, number];
+}
+
+export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor';
+export { extractTextWithPosition as clientExtractTextWithPosition } from './client-extractor';

packages/web-integration/src/extractor/util.ts

@@ -1,5 +1,5 @@
 import SHA256 from 'js-sha256';
-import { extractTextWithPosition } from './extractor';
+import { extractTextWithPosition } from './web-extractor';
 
 // import { TEXT_MAX_SIZE } from './constants';
 let debugMode = false;

packages/web-integration/src/extractor/extractor.ts → packages/web-integration/src/extractor/web-extractor.ts

@@ -3,6 +3,7 @@ import {
   CONTAINER_MINI_WIDTH,
   NodeType,
 } from '@midscene/shared/constants';
+import type { ElementInfo } from '.';
 import {
   isButtonElement,
   isFormElement,
@@ -21,23 +22,6 @@ import {
   visibleRect,
 } from './util';
 
-export interface ElementInfo {
-  id: string;
-  indexId?: string; // for debug use
-  nodePath: string;
-  nodeHashId: string;
-  locator: string;
-  attributes: {
-    nodeType: NodeType;
-    [key: string]: string;
-  };
-  nodeType: NodeType;
-  htmlNode: Node | null;
-  content: string;
-  rect: { left: number; top: number; width: number; height: number };
-  center: [number, number];
-}
-
 function collectElementInfo(node: Node, nodePath: string): ElementInfo | null {
   const rect = visibleRect(node);
   logger('collectElementInfo', node, node.nodeName, rect);

packages/web-integration/src/Page.ts → packages/web-integration/src/page.ts

@@ -1,8 +1,8 @@
 import type { WebKeyInput } from './common/page';
-import type { ElementInfo } from './extractor/extractor';
+import type { ElementInfo } from './extractor';
 
-type encodingType = 'base64' | 'binary';
 type imageType = 'jpeg' | 'png';
+type encodingType = 'base64' | 'binary';
 
 export type screenshotOptions = {
   path?: string;

packages/web-integration/src/playwright/page.ts

@@ -1,9 +1,9 @@
 import type { Page as Browser } from 'playwright';
-import type { AbstractPage, screenshotOptions } from '../Page';
-import type { MouseButton } from '../Page';
 import type { WebKeyInput } from '../common/page';
 import { getExtraReturnLogic } from '../common/utils';
-import type { ElementInfo } from '../extractor/extractor';
+import type { ElementInfo } from '../extractor';
+import type { AbstractPage, screenshotOptions } from '../page';
+import type { MouseButton } from '../page';
 
 export class Page implements AbstractPage {
   private browser: Browser;

packages/web-integration/src/puppeteer/page.ts

@@ -1,9 +1,9 @@
 import type { Page as Browser } from 'puppeteer';
-import type { AbstractPage, screenshotOptions } from '../Page';
-import type { MouseButton } from '../Page';
 import type { WebKeyInput } from '../common/page';
 import { getExtraReturnLogic } from '../common/utils';
-import type { ElementInfo } from '../extractor/extractor';
+import type { ElementInfo } from '../extractor';
+import type { AbstractPage, screenshotOptions } from '../page';
+import type { MouseButton } from '../page';
 
 export class Page implements AbstractPage {
   private browser: Browser;