From 15f04cad771bcb8ce99ebfdbcf9179433b0db8e4 Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Wed, 4 Sep 2024 11:13:05 +0800 Subject: [PATCH] fix: filter unused element for appium --- .vscode/settings.json | 5 +- packages/shared/src/img/transform.ts | 8 ++- packages/web-integration/src/appium/page.ts | 25 +++++-- packages/web-integration/src/common/utils.ts | 2 +- .../web-integration/src/debug/img/util.ts | 2 +- packages/web-integration/src/debug/index.ts | 2 +- ...{appium-exactor.ts => client-extractor.ts} | 72 +++++++++---------- .../web-integration/src/extractor/debug.ts | 6 +- .../web-integration/src/extractor/index.ts | 22 +++++- .../web-integration/src/extractor/util.ts | 2 +- .../{extractor.ts => web-extractor.ts} | 18 +---- .../web-integration/src/{Page.ts => page.ts} | 4 +- .../web-integration/src/playwright/page.ts | 6 +- .../web-integration/src/puppeteer/page.ts | 6 +- 14 files changed, 97 insertions(+), 83 deletions(-) rename packages/web-integration/src/extractor/{appium-exactor.ts => client-extractor.ts} (92%) rename packages/web-integration/src/extractor/{extractor.ts => web-extractor.ts} (94%) rename packages/web-integration/src/{Page.ts => page.ts} (95%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 606f0cac..ad62fffb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,8 +4,5 @@ }, "editor.defaultFormatter": "biomejs.biome", "editor.formatOnSave": true, - "editor.formatOnSaveMode": "modifications", - "[plaintext]": { - "editor.defaultFormatter": "esbenp.prettier-vscode" - } + "editor.formatOnSaveMode": "modifications" } diff --git a/packages/shared/src/img/transform.ts b/packages/shared/src/img/transform.ts index b5ec71bb..2bc0ffdc 100644 --- a/packages/shared/src/img/transform.ts +++ b/packages/shared/src/img/transform.ts @@ -46,6 +46,10 @@ export async function transformImgPathToBase64(inputPath: string) { */ export async function resizeImg( inputData: string | Buffer, + newSize?: { + width: number; + height: number; + }, ): Promise { const isBase64 = typeof inputData === 'string'; const imageBuffer = isBase64 @@ -59,9 +63,9 @@ export async function resizeImg( throw Error('Undefined width or height from the input image.'); } - const newSize = calculateNewDimensions(width, height); + const finalNewSize = newSize || calculateNewDimensions(width, height); - image.resize(newSize.width, newSize.height); + image.resize(finalNewSize.width, finalNewSize.height); const resizedBuffer = await image.getBufferAsync(Jimp.MIME_PNG); return isBase64 ? resizedBuffer.toString('base64') : resizedBuffer; diff --git a/packages/web-integration/src/appium/page.ts b/packages/web-integration/src/appium/page.ts index 7f1d515a..f2d3b953 100644 --- a/packages/web-integration/src/appium/page.ts +++ b/packages/web-integration/src/appium/page.ts @@ -1,9 +1,10 @@ import fs from 'node:fs'; import { resizeImg } from '@midscene/shared/img'; +import { DOMParser } from '@xmldom/xmldom'; import type { KeyInput as PuppeteerKeyInput } from 'puppeteer'; import type { Browser } from 'webdriverio'; -import type { AbstractPage, MouseButton, screenshotOptions } from '../Page'; -import { parsePageSource } from '../extractor/appium-exactor'; +import { clientExtractTextWithPosition } from '../extractor'; +import type { AbstractPage, MouseButton, screenshotOptions } from '../page'; type WebKeyInput = PuppeteerKeyInput; @@ -21,7 +22,19 @@ export class Page implements AbstractPage { async getElementInfos() { const pageSource = await this.browser.getPageSource(); - return parsePageSource(pageSource); + const { width, height } = await this.browser.getWindowSize(); + const parser = new DOMParser(); + const doc = parser.parseFromString(pageSource, 'text/xml'); + const infos = clientExtractTextWithPosition(doc).filter( + (element) => + element.rect.height !== height && + element.rect.width !== width && + element.rect.left !== 0 && + element.rect.top !== 0 && + element.attributes.visible === 'true', + ); + + return infos; } async screenshot(options: screenshotOptions): Promise { @@ -29,8 +42,12 @@ export class Page implements AbstractPage { return Buffer.from(''); } + const { width, height } = await this.browser.getWindowSize(); const screenshotBuffer = await this.browser.saveScreenshot(options.path); - const resizedScreenshotBuffer = await resizeImg(screenshotBuffer); + const resizedScreenshotBuffer = await resizeImg(screenshotBuffer, { + width, + height, + }); if (options.path) { fs.writeFileSync(options.path, resizedScreenshotBuffer); diff --git a/packages/web-integration/src/common/utils.ts b/packages/web-integration/src/common/utils.ts index 4a7b7520..ba9be839 100644 --- a/packages/web-integration/src/common/utils.ts +++ b/packages/web-integration/src/common/utils.ts @@ -2,7 +2,7 @@ import assert from 'node:assert'; import type { Buffer } from 'node:buffer'; import fs, { readFileSync } from 'node:fs'; import path from 'node:path'; -import type { ElementInfo } from '@/extractor/extractor'; +import type { ElementInfo } from '@/extractor'; import type { PlaywrightParserOpt, UIContext } from '@midscene/core'; import { getTmpFile } from '@midscene/core/utils'; import { base64Encoded, imageInfoOfBase64 } from '@midscene/shared/img'; diff --git a/packages/web-integration/src/debug/img/util.ts b/packages/web-integration/src/debug/img/util.ts index ef372509..36fdd7b2 100644 --- a/packages/web-integration/src/debug/img/util.ts +++ b/packages/web-integration/src/debug/img/util.ts @@ -1,5 +1,5 @@ +import type { ElementInfo } from '@/extractor'; import { NodeType } from '@/extractor/constants'; -import type { ElementInfo } from '@/extractor/extractor'; import type { WebPage } from '../../common/page'; export async function getElementInfos(page: WebPage) { diff --git a/packages/web-integration/src/debug/index.ts b/packages/web-integration/src/debug/index.ts index 980bc2d2..07b195ab 100644 --- a/packages/web-integration/src/debug/index.ts +++ b/packages/web-integration/src/debug/index.ts @@ -1,8 +1,8 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import type { WebPage } from '@/common/page'; +import type { ElementInfo } from '@/extractor'; import { NodeType } from '@/extractor/constants'; -import type { ElementInfo } from '@/extractor/extractor'; import { getTmpFile } from '@midscene/core/utils'; import { processImageElementInfo, diff --git a/packages/web-integration/src/extractor/appium-exactor.ts b/packages/web-integration/src/extractor/client-extractor.ts similarity index 92% rename from packages/web-integration/src/extractor/appium-exactor.ts rename to packages/web-integration/src/extractor/client-extractor.ts index 11d2f6a9..eca891ae 100644 --- a/packages/web-integration/src/extractor/appium-exactor.ts +++ b/packages/web-integration/src/extractor/client-extractor.ts @@ -1,6 +1,5 @@ -import { DOMParser } from '@xmldom/xmldom'; +import type { ElementInfo } from './'; import { NodeType } from './constants'; -import type { ElementInfo } from './extractor'; import { generateId, midsceneGenerateHash } from './util'; // https://github.com/appium/appium/tree/master/packages/universal-xml-plugin @@ -55,11 +54,36 @@ function validTextNodeContent(node: Node): string { return ''; } -// New parsePageSource function to extract from Appium's pageSource -export function parsePageSource(pageSource: string) { - const parser = new DOMParser(); - const doc = parser.parseFromString(pageSource, 'text/xml'); - return extractTextWithPosition(doc); +function getXPathForElement(element: Node): string { + if (element.nodeType !== 1) { + return ''; + } + + const getIndex = (sib: Node, name: string) => { + let count = 1; + for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) { + if (cur.nodeType === 1 && cur.nodeName === name) { + count++; + } + } + return count; + }; + + const getPath = (node: Node, path = ''): string => { + if (node.parentNode) { + path = getPath(node.parentNode, path); + } + + if (node.nodeType === 1) { + const index = getIndex(node, node.nodeName); + const tagName = node.nodeName.toLowerCase(); + path += `/${tagName}[${index}]`; + } + + return path; + }; + + return getPath(element); } // Perform DFS traversal and collect element information @@ -107,7 +131,6 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] { } const xpath = getXPathForElement(node); - const elementInfo: ElementInfo = { id: nodeHashId, indexId: generateId(nodeIndex++), @@ -125,6 +148,7 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] { ], nodeType, htmlNode: null, + nodePath: '', }; elementInfoArray.push(elementInfo); @@ -136,35 +160,3 @@ export function extractTextWithPosition(initNode: Document): ElementInfo[] { return elementInfoArray; } - -function getXPathForElement(element: Node): string { - if (element.nodeType !== 1) { - return ''; - } - - const getIndex = (sib: Node, name: string) => { - let count = 1; - for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) { - if (cur.nodeType === 1 && cur.nodeName === name) { - count++; - } - } - return count; - }; - - const getPath = (node: Node, path = ''): string => { - if (node.parentNode) { - path = getPath(node.parentNode, path); - } - - if (node.nodeType === 1) { - const index = getIndex(node, node.nodeName); - const tagName = node.nodeName.toLowerCase(); - path += `/${tagName}[${index}]`; - } - - return path; - }; - - return getPath(element); -} diff --git a/packages/web-integration/src/extractor/debug.ts b/packages/web-integration/src/extractor/debug.ts index 9699e960..5afe7a78 100644 --- a/packages/web-integration/src/extractor/debug.ts +++ b/packages/web-integration/src/extractor/debug.ts @@ -1,6 +1,6 @@ -import { extractTextWithPosition } from '.'; +import { webExtractTextWithPosition } from '.'; import { setExtractTextWithPositionOnWindow } from './util'; -console.log(extractTextWithPosition(document.body, true)); -console.log(JSON.stringify(extractTextWithPosition(document.body, false))); +console.log(webExtractTextWithPosition(document.body, true)); +console.log(JSON.stringify(webExtractTextWithPosition(document.body, false))); setExtractTextWithPositionOnWindow(); diff --git a/packages/web-integration/src/extractor/index.ts b/packages/web-integration/src/extractor/index.ts index cc63aa19..cf94563f 100644 --- a/packages/web-integration/src/extractor/index.ts +++ b/packages/web-integration/src/extractor/index.ts @@ -1 +1,21 @@ -export { extractTextWithPosition } from './extractor'; +import type { NodeType } from '@midscene/shared/constants'; + +export interface ElementInfo { + id: string; + indexId?: string; // for debug use + nodePath: string; + nodeHashId: string; + locator: string; + attributes: { + nodeType: NodeType; + [key: string]: string; + }; + nodeType: NodeType; + htmlNode: Node | null; + content: string; + rect: { left: number; top: number; width: number; height: number }; + center: [number, number]; +} + +export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor'; +export { extractTextWithPosition as clientExtractTextWithPosition } from './client-extractor'; diff --git a/packages/web-integration/src/extractor/util.ts b/packages/web-integration/src/extractor/util.ts index d9597c7c..56bba39b 100644 --- a/packages/web-integration/src/extractor/util.ts +++ b/packages/web-integration/src/extractor/util.ts @@ -1,5 +1,5 @@ import SHA256 from 'js-sha256'; -import { extractTextWithPosition } from './extractor'; +import { extractTextWithPosition } from './web-extractor'; // import { TEXT_MAX_SIZE } from './constants'; let debugMode = false; diff --git a/packages/web-integration/src/extractor/extractor.ts b/packages/web-integration/src/extractor/web-extractor.ts similarity index 94% rename from packages/web-integration/src/extractor/extractor.ts rename to packages/web-integration/src/extractor/web-extractor.ts index 4a81e625..65e79801 100644 --- a/packages/web-integration/src/extractor/extractor.ts +++ b/packages/web-integration/src/extractor/web-extractor.ts @@ -3,6 +3,7 @@ import { CONTAINER_MINI_WIDTH, NodeType, } from '@midscene/shared/constants'; +import type { ElementInfo } from '.'; import { isButtonElement, isFormElement, @@ -21,23 +22,6 @@ import { visibleRect, } from './util'; -export interface ElementInfo { - id: string; - indexId?: string; // for debug use - nodePath: string; - nodeHashId: string; - locator: string; - attributes: { - nodeType: NodeType; - [key: string]: string; - }; - nodeType: NodeType; - htmlNode: Node | null; - content: string; - rect: { left: number; top: number; width: number; height: number }; - center: [number, number]; -} - function collectElementInfo(node: Node, nodePath: string): ElementInfo | null { const rect = visibleRect(node); logger('collectElementInfo', node, node.nodeName, rect); diff --git a/packages/web-integration/src/Page.ts b/packages/web-integration/src/page.ts similarity index 95% rename from packages/web-integration/src/Page.ts rename to packages/web-integration/src/page.ts index fe465d06..9f0ae960 100644 --- a/packages/web-integration/src/Page.ts +++ b/packages/web-integration/src/page.ts @@ -1,8 +1,8 @@ import type { WebKeyInput } from './common/page'; -import type { ElementInfo } from './extractor/extractor'; +import type { ElementInfo } from './extractor'; -type encodingType = 'base64' | 'binary'; type imageType = 'jpeg' | 'png'; +type encodingType = 'base64' | 'binary'; export type screenshotOptions = { path?: string; diff --git a/packages/web-integration/src/playwright/page.ts b/packages/web-integration/src/playwright/page.ts index e769766c..b9e4965e 100644 --- a/packages/web-integration/src/playwright/page.ts +++ b/packages/web-integration/src/playwright/page.ts @@ -1,9 +1,9 @@ import type { Page as Browser } from 'playwright'; -import type { AbstractPage, screenshotOptions } from '../Page'; -import type { MouseButton } from '../Page'; import type { WebKeyInput } from '../common/page'; import { getExtraReturnLogic } from '../common/utils'; -import type { ElementInfo } from '../extractor/extractor'; +import type { ElementInfo } from '../extractor'; +import type { AbstractPage, screenshotOptions } from '../page'; +import type { MouseButton } from '../page'; export class Page implements AbstractPage { private browser: Browser; diff --git a/packages/web-integration/src/puppeteer/page.ts b/packages/web-integration/src/puppeteer/page.ts index d3c41ec0..7f68312e 100644 --- a/packages/web-integration/src/puppeteer/page.ts +++ b/packages/web-integration/src/puppeteer/page.ts @@ -1,9 +1,9 @@ import type { Page as Browser } from 'puppeteer'; -import type { AbstractPage, screenshotOptions } from '../Page'; -import type { MouseButton } from '../Page'; import type { WebKeyInput } from '../common/page'; import { getExtraReturnLogic } from '../common/utils'; -import type { ElementInfo } from '../extractor/extractor'; +import type { ElementInfo } from '../extractor'; +import type { AbstractPage, screenshotOptions } from '../page'; +import type { MouseButton } from '../page'; export class Page implements AbstractPage { private browser: Browser;