From e9d4e903e3ec6de6c6f2706456ed6f9ce691d093 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 13:49:20 +0200 Subject: [PATCH 01/21] feat: Add 'mobile: calibrateWebToRealCoordinatesTranslation' API --- docs/endpoints.md | 1 - docs/execute-methods.md | 19 +++ lib/commands/context.js | 1 - lib/commands/gesture.js | 50 ------ lib/commands/proxy-helper.js | 3 - lib/commands/web.js | 150 ++++++++++++------ lib/driver.js | 19 +-- lib/execute-method-map.ts | 3 + lib/method-map.js | 6 - lib/types.ts | 11 ++ package.json | 2 +- test/functional/web/safari-basic-e2e-specs.js | 6 - 12 files changed, 137 insertions(+), 134 deletions(-) diff --git a/docs/endpoints.md b/docs/endpoints.md index fb0384b45..f512f7c8d 100644 --- a/docs/endpoints.md +++ b/docs/endpoints.md @@ -65,7 +65,6 @@ | POST | /alert_text | text | | | POST | /accept_alert | none | | | POST | /dismiss_alert | none | | -| POST | /moveto | | element, xoffset, yoffset | | POST | /click | | button | | POST | /touch/click | element | | | POST | /touch/flick | | element, xspeed, yspeed, xoffset, yoffset, speed | diff --git a/docs/execute-methods.md b/docs/execute-methods.md index 08417980b..f5ab1b2a1 100644 --- a/docs/execute-methods.md +++ b/docs/execute-methods.md @@ -1201,6 +1201,25 @@ If the connection is disconnected, condition inducer will be automatically disab Either `true` or `false`, where `true` means disabling of the condition inducer has been successful +### mobile: calibrateWebToRealCoordinatesTranslation + +Calibrates web to real coordinates translation. +This API can only be called from Safari web context. +It must load a custom page to the browser, and then restore +the original one, so don't call it if you can potentially +lose the current web app state. +The outcome of this API is then used in `nativeWebTap` mode. +The returned value could also be used to manually transform web coordinates +to real devices ones in client scripts. + +It is adviced to call this API at least once before changing the device orientation +or device screen layout as the recetly received value is cached for the session lifetime +and may become obsolete. + +#### Returned Result + +An object with a single `offset` property. The property has two subproperties: `dx` and `dy` used to properly shift Safari web element coordinates into native context. + ### mobile: updateSafariPreferences Updates preferences of Mobile Safari on Simulator diff --git a/lib/commands/context.js b/lib/commands/context.js index 2aec1633d..9e4297e01 100644 --- a/lib/commands/context.js +++ b/lib/commands/context.js @@ -392,7 +392,6 @@ const helpers = { await this.remote.disconnect(); this.curContext = null; this.curWebFrames = []; - this.curWebCoords = null; this.remote = null; }, /** diff --git a/lib/commands/gesture.js b/lib/commands/gesture.js index 6ce58a797..63b3e0600 100644 --- a/lib/commands/gesture.js +++ b/lib/commands/gesture.js @@ -97,43 +97,6 @@ export function gesturesChainToString(gestures, keysToInclude = ['options']) { } const commands = { - /** - * Move the mouse pointer to a particular screen location - * - * @param {string|Element} el - the element ID if the move is relative to an element - * @param {number} xoffset - the x offset - * @param {number} yoffset - the y offset - * @this {XCUITestDriver} - * @deprecated Use {@linkcode XCUITestDriver.performActions} instead - */ - async moveTo(el, xoffset = 0, yoffset = 0) { - el = util.unwrapElement(el); - - if (this.isWebContext()) { - throw new errors.UnknownMethodError( - 'The moveTo command is not available in the web context. Use the Actions API in the ' + - 'native context instead', - ); - } else { - if (_.isNil(el)) { - if (!this.curCoords) { - throw new errors.UnknownError( - 'Current cursor position unknown, please use moveTo with an element the first time.', - ); - } - this.curCoords = { - x: this.curCoords.x + xoffset, - y: this.curCoords.y + yoffset, - }; - } else { - let elPos = await this.getLocation(el); - this.curCoords = { - x: elPos.x + xoffset, - y: elPos.y + yoffset, - }; - } - } - }, /** * Shake the device * @this {XCUITestDriver} @@ -767,19 +730,6 @@ const helpers = { } return coordinates; }, - /** - * @this {XCUITestDriver} - */ - applyMoveToOffset(firstCoordinates, secondCoordinates) { - if (secondCoordinates.areOffsets) { - return { - x: firstCoordinates.x + secondCoordinates.x, - y: firstCoordinates.y + secondCoordinates.y, - }; - } else { - return secondCoordinates; - } - }, }; export default {...helpers, ...commands}; diff --git a/lib/commands/proxy-helper.js b/lib/commands/proxy-helper.js index 63edec58c..3b6880b9d 100644 --- a/lib/commands/proxy-helper.js +++ b/lib/commands/proxy-helper.js @@ -49,9 +49,6 @@ const WDA_ROUTES = /** @type {const} */ ({ '/wda/locked': { GET: 'isLocked', }, - '/wda/tap/nil': { - POST: 'clickCoords', - }, '/window/size': { GET: 'getWindowSize', }, diff --git a/lib/commands/web.js b/lib/commands/web.js index ba9183eef..2aa065d13 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -82,11 +82,7 @@ async function tapWebElementNatively(driver, atomsElement) { return false; } } - const coords = { - x: Math.round(rect.x + rect.width / 2), - y: Math.round(rect.y + rect.height / 2), - }; - await driver.clickCoords(coords); + await driver.mobileTap(Math.round(rect.x + rect.width / 2), Math.round(rect.y + rect.height / 2)); return true; } } @@ -435,10 +431,12 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @param {number} x + * @param {number} y */ - async clickWebCoords() { - let coords = await this.translateWebCoords(this.curWebCoords); - await this.clickCoords(coords); + async clickWebCoords(x, y) { + const {x: translatedX, y: translatedY} = await this.translateWebCoords(x, y); + await this.mobileTap(translatedX, translatedY); }, /** * @this {XCUITestDriver} @@ -645,29 +643,30 @@ const extensions = { ]) ); const {width, height} = size; - let {x, y} = coordinates; - x += width / 2; - y += height / 2; - - this.curWebCoords = {x, y}; - await this.clickWebCoords(); + const {x, y} = coordinates; + await this.clickWebCoords(x + width / 2, y + height / 2); }, /** * @this {XCUITestDriver} - */ - async clickCoords(coords) { - await this.performTouch([ - { - action: 'tap', - options: coords, - }, - ]); - }, - /** - * @this {XCUITestDriver} - */ - async translateWebCoords(coords) { - this.log.debug(`Translating coordinates (${JSON.stringify(coords)}) to web coordinates`); + * @param {number} x + * @param {number} y + * @returns {Promise} + */ + async translateWebCoords(x, y) { + this.log.debug(`Translating web coordinates (${JSON.stringify({x, y})}) to native coordinates`); + + if (this.webviewOffset) { + this.log.debug(`Will use the previously calibrated offset: ${JSON.stringify(this.webviewOffset)}`); + return { + x: x + this.webviewOffset.dx, + y: y + this.webviewOffset.dy, + }; + } else { + this.log.debug( + `Using the legacy algorithm for coordinates translation. ` + + `Invoke 'mobile: calibrateWebToRealCoordinatesTranslation' to change that.` + ); + } // absolutize web coords /** @type {import('@appium/types').Element|undefined|string} */ @@ -705,32 +704,37 @@ const extensions = { } finally { this.setImplicitWait(implicitWaitMs); } - - if (wvDims && realDims && wvPos) { - let xRatio = realDims.w / wvDims.w; - let yRatio = realDims.h / wvDims.h; - let newCoords = { - x: wvPos.x + Math.round(xRatio * coords.x), - y: wvPos.y + Math.round(yRatio * coords.y), - }; - - // additional logging for coordinates, since it is sometimes broken - // see https://github.com/appium/appium/issues/9159 - this.log.debug(`Converted coordinates: ${JSON.stringify(newCoords)}`); - this.log.debug(` rect: ${JSON.stringify(rect)}`); - this.log.debug(` wvPos: ${JSON.stringify(wvPos)}`); - this.log.debug(` realDims: ${JSON.stringify(realDims)}`); - this.log.debug(` wvDims: ${JSON.stringify(wvDims)}`); - this.log.debug(` xRatio: ${JSON.stringify(xRatio)}`); - this.log.debug(` yRatio: ${JSON.stringify(yRatio)}`); - - this.log.debug( - `Converted web coords ${JSON.stringify(coords)} into real coords ${JSON.stringify( - newCoords, - )}`, + if (!wvDims || !realDims || !wvPos) { + throw new Error( + `Web coordinates ${JSON.stringify({x, y})} cannot be translated into real coordinates. ` + + `Try to invoke 'mobile: calibrateWebToRealCoordinatesTranslation' or consider translating the ` + + `coordinates from the client code.` ); - return newCoords; } + + const xRatio = realDims.w / wvDims.w; + const yRatio = realDims.h / wvDims.h; + const newCoords = { + x: wvPos.x + Math.round(xRatio * x), + y: wvPos.y + Math.round(yRatio * y), + }; + + // additional logging for coordinates, since it is sometimes broken + // see https://github.com/appium/appium/issues/9159 + this.log.debug(`Converted coordinates: ${JSON.stringify(newCoords)}`); + this.log.debug(` rect: ${JSON.stringify(rect)}`); + this.log.debug(` wvPos: ${JSON.stringify(wvPos)}`); + this.log.debug(` realDims: ${JSON.stringify(realDims)}`); + this.log.debug(` wvDims: ${JSON.stringify(wvDims)}`); + this.log.debug(` xRatio: ${JSON.stringify(xRatio)}`); + this.log.debug(` yRatio: ${JSON.stringify(yRatio)}`); + + this.log.debug( + `Converted web coords ${JSON.stringify({x, y})} into real coords ${JSON.stringify( + newCoords, + )}`, + ); + return newCoords; }, /** * @this {XCUITestDriver} @@ -834,6 +838,48 @@ const extensions = { } }, + /** + * Calibrates web to real coordinates translation. + * This API can only be called from Safari web context. + * It must load a custom page to the browser, and then restore + * the original one, so don't call it if you can potentially + * lose the current web app state. + * The outcome of this API is then used in nativeWebTap mode. + * The returned value could also be used to manually transform web coordinates + * to real devices ones in client scripts. + * + * @this {XCUITestDriver} + * @returns {Promise<{offset: import('../types').Delta}>} + */ + async mobileCalibrateWebToRealCoordinatesTranslation() { + if (!this.isWebContext()) { + throw new errors.NotImplementedError('This API can only be called from a web context'); + } + + const currentUrl = await this.getUrl(); + await this.setUrl(`http://127.0.0.1:${this.opts.wdaLocalPort || 8100}/calibrate`); + const {width, height} = await this.getWindowRect(); + const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; + await this.mobileTap(centerX, centerY); + /** @type {import('../types').Delta} */ + try { + this.webviewOffset = JSON.parse(await this.title()); + } catch (e) { + throw new Error( + `Cannot determine web view coordinates offset. Are you in Safari context? ` + + `Original error: ${e.message}` + ); + } + if (currentUrl) { + // restore the previous url + await this.setUrl(currentUrl); + } + return { + // @ts-ignore webviewOffset is always defined here + offset: this.webviewOffset + }; + }, + /** * @typedef {Object} SafariOpts * @property {object} preferences An object containing Safari settings to be updated. diff --git a/lib/driver.js b/lib/driver.js index 20ad1eb0e..d59b3f41e 100644 --- a/lib/driver.js +++ b/lib/driver.js @@ -141,7 +141,6 @@ const NO_PROXY_NATIVE_LIST = [ ['POST', /execute/], ['POST', /keys/], ['POST', /log/], - ['POST', /moveto/], ['POST', /receive_async_response/], // always, in case context switches while waiting ['POST', /session\/[^\/]+\/location/], // geo location, but not element location ['POST', /shake/], @@ -199,19 +198,12 @@ class XCUITestDriver extends BaseDriver { /** @type {string|null} */ curContext; - /** - * @type {import('@appium/types').Position|null} - */ - curWebCoords; - - /** - * @type {import('@appium/types').Position|null} - */ - curCoords; - /** @type {string[]} */ curWebFrames; + /** @type {import('./types').Delta|null} */ + webviewOffset; + /** * @type {import('./types').Page[]|undefined} */ @@ -293,6 +285,7 @@ class XCUITestDriver extends BaseDriver { this.webElementsCache = new LRUCache({ max: WEB_ELEMENTS_CACHE_SIZE, }); + this.webviewOffset = null; this._waitingAtoms = { count: 0, alertNotifier: new EventEmitter(), @@ -1998,7 +1991,6 @@ class XCUITestDriver extends BaseDriver { /*---------+ | GESTURE | +---------+*/ - moveTo = commands.gestureExtensions.moveTo; mobileShake = commands.gestureExtensions.mobileShake; click = commands.gestureExtensions.click; releaseActions = commands.gestureExtensions.releaseActions; @@ -2021,7 +2013,6 @@ class XCUITestDriver extends BaseDriver { mobileSelectPickerWheelValue = commands.gestureExtensions.mobileSelectPickerWheelValue; mobileRotateElement = commands.gestureExtensions.mobileRotateElement; getCoordinates = commands.gestureExtensions.getCoordinates; - applyMoveToOffset = commands.gestureExtensions.applyMoveToOffset; /*-------+ | IOHID | @@ -2198,8 +2189,8 @@ class XCUITestDriver extends BaseDriver { checkForAlert = commands.webExtensions.checkForAlert; waitForAtom = commands.webExtensions.waitForAtom; mobileWebNav = commands.webExtensions.mobileWebNav; + mobileCalibrateWebToRealCoordinatesTranslation = commands.webExtensions.mobileCalibrateWebToRealCoordinatesTranslation; mobileUpdateSafariPreferences = commands.webExtensions.mobileUpdateSafariPreferences; - clickCoords = commands.webExtensions.clickCoords; /*--------+ | XCTEST | diff --git a/lib/execute-method-map.ts b/lib/execute-method-map.ts index fe7259839..a68b0230e 100644 --- a/lib/execute-method-map.ts +++ b/lib/execute-method-map.ts @@ -439,6 +439,9 @@ export const executeMethodMap = { required: ['preferences'], }, }, + 'mobile: calibrateWebToRealCoordinatesTranslation': { + command: 'mobileCalibrateWebToRealCoordinatesTranslation', + }, 'mobile: deepLink': { command: 'mobileDeepLink', params: { diff --git a/lib/method-map.js b/lib/method-map.js index db1ad7451..3b032eb9a 100644 --- a/lib/method-map.js +++ b/lib/method-map.js @@ -16,12 +16,6 @@ export const newMethodMap = /** @type {const} */ ({ '/session/:sessionId/element/:elementId/location': {GET: {command: 'getLocation'}}, '/session/:sessionId/element/:elementId/location_in_view': {GET: {command: 'getLocationInView'}}, '/session/:sessionId/element/:elementId/size': {GET: {command: 'getSize'}}, - '/session/:sessionId/moveto': { - POST: { - command: 'moveTo', - payloadParams: {optional: ['element', 'xoffset', 'yoffset']}, - }, - }, '/session/:sessionId/touch/click': { POST: {command: 'click', payloadParams: {required: ['element']}}, }, diff --git a/lib/types.ts b/lib/types.ts index e927762be..ccbfd62e3 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -65,3 +65,14 @@ export interface WDACapabilities { defaultAlertAction: 'accept' | 'dismiss'; capabilities?: StringRecord; } + +export interface Delta { + /** + * x offset + */ + dx: number; + /** + * y offset + */ + dy: number; +} diff --git a/package.json b/package.json index ef568ca10..274670c26 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "appium-ios-device": "^2.5.4", "appium-ios-simulator": "^5.3.3", "appium-remote-debugger": "^10.0.0", - "appium-webdriveragent": "^5.9.1", + "appium-webdriveragent": "^5.11.0", "appium-xcode": "^5.1.4", "async-lock": "^1.4.0", "asyncbox": "^2.9.4", diff --git a/test/functional/web/safari-basic-e2e-specs.js b/test/functional/web/safari-basic-e2e-specs.js index b13d1458c..251dda70a 100644 --- a/test/functional/web/safari-basic-e2e-specs.js +++ b/test/functional/web/safari-basic-e2e-specs.js @@ -262,12 +262,6 @@ describe('Safari - basics -', function () { size.height.should.be.above(0); size.width.should.be.above(0); }); - it.skip('should move to an arbitrary x-y element and click on it', async function () { - const el = await driver.$('=i am a link'); - await driver.moveTo(el, 5, 15); - await el.click(); - await spinTitleEquals(driver, 'I am another page title'); - }); // TODO: Update for WdIO compatibility it.skip('should submit a form', async function () { const el = await driver.$('#comments'); From 26bb8bbb9806af71604501a5b570d939451f5458 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 13:52:16 +0200 Subject: [PATCH 02/21] Update lock --- package-lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 08372214a..598f4f1f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "appium-ios-device": "^2.5.4", "appium-ios-simulator": "^5.3.3", "appium-remote-debugger": "^10.0.0", - "appium-webdriveragent": "^5.9.1", + "appium-webdriveragent": "^5.11.0", "appium-xcode": "^5.1.4", "async-lock": "^1.4.0", "asyncbox": "^2.9.4", @@ -4283,9 +4283,9 @@ } }, "node_modules/appium-webdriveragent": { - "version": "5.10.1", - "resolved": "https://registry.npmjs.org/appium-webdriveragent/-/appium-webdriveragent-5.10.1.tgz", - "integrity": "sha512-iMrSHCncnlXocM8E08ov+1fcPPar01Zoava1SKM8Yij3HUaropeOA1ccID8VWnXrV6H3HK7GHR7s3K7eQhJCdA==", + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/appium-webdriveragent/-/appium-webdriveragent-5.11.0.tgz", + "integrity": "sha512-vBKcGvs/KaQ5Gngs1rcOfapubrNOEi8v3gmyX+45vpKo0tBHjJ+j1ZtkHuxDd3AymXw0SZLYO0RfNRzYCvtIKw==", "dependencies": { "@appium/base-driver": "^9.0.0", "@appium/strongbox": "^0.x", From 41b5a676486958120164d36107ffa826469c6ba4 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 18:17:37 +0200 Subject: [PATCH 03/21] fix delta calculation --- lib/commands/web.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 2aa065d13..5a5ce4db2 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -862,14 +862,21 @@ const extensions = { const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; await this.mobileTap(centerX, centerY); /** @type {import('../types').Delta} */ + let webClickCoordinates; + const errorPrefix = `Cannot determine web view coordinates offset. Are you in Safari context?`; try { - this.webviewOffset = JSON.parse(await this.title()); + webClickCoordinates = JSON.parse(await this.title()); } catch (e) { - throw new Error( - `Cannot determine web view coordinates offset. Are you in Safari context? ` + - `Original error: ${e.message}` - ); + throw new Error(`${errorPrefix} Original error: ${e.message}`); } + const {dx, dy} = webClickCoordinates; + if (!_.isInteger(dx) || !_.isInteger(dy)) { + throw new Error(errorPrefix); + } + this.webviewOffset = { + dx: centerX - dx, + dy: centerY - dy, + }; if (currentUrl) { // restore the previous url await this.setUrl(currentUrl); From b32cc31672c36a773456060d067290b792c4cc4e Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 18:30:44 +0200 Subject: [PATCH 04/21] fixes --- lib/commands/web.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 5a5ce4db2..1d3be21da 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -861,21 +861,23 @@ const extensions = { const {width, height} = await this.getWindowRect(); const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; await this.mobileTap(centerX, centerY); - /** @type {import('../types').Delta} */ + const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?'; let webClickCoordinates; - const errorPrefix = `Cannot determine web view coordinates offset. Are you in Safari context?`; + let title; try { - webClickCoordinates = JSON.parse(await this.title()); + title = await this.title(); + this.log.debug(JSON.stringify(title)); + webClickCoordinates = JSON.parse(title); } catch (e) { throw new Error(`${errorPrefix} Original error: ${e.message}`); } - const {dx, dy} = webClickCoordinates; - if (!_.isInteger(dx) || !_.isInteger(dy)) { + const {x, y} = webClickCoordinates; + if (!_.isInteger(x) || !_.isInteger(y)) { throw new Error(errorPrefix); } this.webviewOffset = { - dx: centerX - dx, - dy: centerY - dy, + dx: centerX - x, + dy: centerY - y, }; if (currentUrl) { // restore the previous url From 1fd0ec92d22084689a1646e5c9ca3f11156962c9 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 18:33:22 +0200 Subject: [PATCH 05/21] Typing --- lib/commands/web.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 1d3be21da..59dc58ef8 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -883,10 +883,7 @@ const extensions = { // restore the previous url await this.setUrl(currentUrl); } - return { - // @ts-ignore webviewOffset is always defined here - offset: this.webviewOffset - }; + return {offset: this.webviewOffset}; }, /** From 7fa4ca82707056c3ded96d3086f3e1e73da40084 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 19:04:40 +0200 Subject: [PATCH 06/21] Address comments --- docs/capabilities.md | 1 + lib/commands/web.js | 16 +++++++++++++++- lib/desired-caps.js | 3 +++ lib/driver.js | 3 ++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/capabilities.md b/docs/capabilities.md index 8bbcfb33e..f3f25559b 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -52,6 +52,7 @@ Capability | Description |`appium:wdaStartupRetries`|Number of times to try to build and launch `WebDriverAgent` onto the device. Defaults to 2.|e.g., `4`| |`appium:wdaStartupRetryInterval`|Time, in ms, to wait between tries to build and launch `WebDriverAgent`. Defaults to 10000ms.|e.g., `20000`| |`appium:wdaLocalPort`|This value if specified, will be used to forward traffic from Mac host to real ios devices over USB. Default value is same as port number used by WDA on device.|e.g., `8100`| +|`appium:wdaRemotePort`|This value if specified, will be used as the port number to start WDA HTTP server on the remote device. This is only relevant for real devices, because Simulator shares ports with its host. If `webDriverAgentUrl` is provided then it might be used to provide a hint for the remote port number if it differs from the default one. Default value is 8100.|e.g., `8100`| |`appium:wdaBaseUrl`| This value if specified, will be used as a prefix to build a custom `WebDriverAgent` url. It is different from `webDriverAgentUrl`, because if the latter is set then it expects `WebDriverAgent` to be already listening and skips the building phase. Defaults to `http://localhost` | e.g., `http://192.168.1.100`| |`appium:showXcodeLog`|Whether to display the output of the Xcode command used to run the tests. If this is `true`, there will be **lots** of extra logging at startup. Defaults to `false`|e.g., `true`| |`appium:iosInstallPause`|Time in milliseconds to pause between installing the application and starting `WebDriverAgent` on the device. Used particularly for larger applications. Defaults to `0`|e.g., `8000`| diff --git a/lib/commands/web.js b/lib/commands/web.js index 59dc58ef8..8932d8180 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -838,6 +838,20 @@ const extensions = { } }, + /** + * @this {XCUITestDriver} + * @returns {string} The base url which could be used to access WDA HTTP endpoints + * FROM THE SAME DEVICE where WDA is running + */ + getWdaLocalhostRoot() { + const remotePort = + (this.opts.wdaRemotePort + ?? this.wda?.wdaRemotePort + ?? this.opts.wdaLocalPort) + || 8100; + return `http://127.0.0.1:${remotePort}`; + }, + /** * Calibrates web to real coordinates translation. * This API can only be called from Safari web context. @@ -857,7 +871,7 @@ const extensions = { } const currentUrl = await this.getUrl(); - await this.setUrl(`http://127.0.0.1:${this.opts.wdaLocalPort || 8100}/calibrate`); + await this.setUrl(`${this.getWdaLocalhostRoot()}/calibrate`); const {width, height} = await this.getWindowRect(); const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; await this.mobileTap(centerX, centerY); diff --git a/lib/desired-caps.js b/lib/desired-caps.js index 957a73bf9..f04bd29b3 100644 --- a/lib/desired-caps.js +++ b/lib/desired-caps.js @@ -103,6 +103,9 @@ const desiredCapConstraints = /** @type {const} */ ({ wdaLocalPort: { isNumber: true, }, + wdaRemotePort: { + isNumber: true, + }, wdaBaseUrl: { isString: true, }, diff --git a/lib/driver.js b/lib/driver.js index d59b3f41e..6fdec4602 100644 --- a/lib/driver.js +++ b/lib/driver.js @@ -468,7 +468,7 @@ class XCUITestDriver extends BaseDriver { */ getDefaultUrl() { // Setting this to some external URL slows down the session init - return `http://127.0.0.1:${this.opts.wdaLocalPort || 8100}/health`; + return `${this.getWdaLocalhostRoot()}/health`; } async start() { @@ -2189,6 +2189,7 @@ class XCUITestDriver extends BaseDriver { checkForAlert = commands.webExtensions.checkForAlert; waitForAtom = commands.webExtensions.waitForAtom; mobileWebNav = commands.webExtensions.mobileWebNav; + getWdaLocalhostRoot = commands.webExtensions.getWdaLocalhostRoot; mobileCalibrateWebToRealCoordinatesTranslation = commands.webExtensions.mobileCalibrateWebToRealCoordinatesTranslation; mobileUpdateSafariPreferences = commands.webExtensions.mobileUpdateSafariPreferences; From d23b22f048a535509d7fa038d67cc27140ba88c5 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 21:11:23 +0200 Subject: [PATCH 07/21] Tune port detection --- lib/commands/web.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 8932d8180..e1b850cd0 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -846,8 +846,8 @@ const extensions = { getWdaLocalhostRoot() { const remotePort = (this.opts.wdaRemotePort - ?? this.wda?.wdaRemotePort - ?? this.opts.wdaLocalPort) + ?? this.opts.wdaLocalPort + ?? this.wda?.url?.port) || 8100; return `http://127.0.0.1:${remotePort}`; }, From fa93b013fc31e9f0c6723015553ba02240d19208 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 6 Oct 2023 21:15:18 +0200 Subject: [PATCH 08/21] moar --- lib/commands/web.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index e1b850cd0..c050640a1 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -846,8 +846,8 @@ const extensions = { getWdaLocalhostRoot() { const remotePort = (this.opts.wdaRemotePort - ?? this.opts.wdaLocalPort - ?? this.wda?.url?.port) + ?? this.wda?.url?.port + ?? this.opts.wdaLocalPort) || 8100; return `http://127.0.0.1:${remotePort}`; }, From 1156344dd936f271aa56cdf0d18288a0db0260fc Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 7 Oct 2023 14:16:13 +0200 Subject: [PATCH 09/21] Properly fetch native size --- lib/commands/web.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index c050640a1..b1e31c50b 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -872,7 +872,7 @@ const extensions = { const currentUrl = await this.getUrl(); await this.setUrl(`${this.getWdaLocalhostRoot()}/calibrate`); - const {width, height} = await this.getWindowRect(); + const {width, height} = /** @type {import('@appium/types').Size} */(await this.proxyCommand(`/window/size`, 'GET')); const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; await this.mobileTap(centerX, centerY); const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?'; From b49fcc4ba044d6537e0c9caae2ada675a6bc9dfd Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 7 Oct 2023 22:30:14 +0200 Subject: [PATCH 10/21] Add retry --- lib/commands/web.js | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index b1e31c50b..1d04069ed 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -874,29 +874,38 @@ const extensions = { await this.setUrl(`${this.getWdaLocalhostRoot()}/calibrate`); const {width, height} = /** @type {import('@appium/types').Size} */(await this.proxyCommand(`/window/size`, 'GET')); const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; - await this.mobileTap(centerX, centerY); const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?'; - let webClickCoordinates; - let title; - try { - title = await this.title(); - this.log.debug(JSON.stringify(title)); - webClickCoordinates = JSON.parse(title); - } catch (e) { - throw new Error(`${errorPrefix} Original error: ${e.message}`); - } - const {x, y} = webClickCoordinates; - if (!_.isInteger(x) || !_.isInteger(y)) { - throw new Error(errorPrefix); - } - this.webviewOffset = { - dx: centerX - x, - dy: centerY - y, - }; + + await retryInterval( + 10, + 500, + async () => { + await this.mobileTap(centerX, centerY); + let webClickCoordinates; + let title; + try { + title = await this.title(); + this.log.debug(JSON.stringify(title)); + webClickCoordinates = JSON.parse(title); + } catch (e) { + throw new Error(`${errorPrefix} Original error: ${e.message}`); + } + const {x, y} = webClickCoordinates; + if (!_.isInteger(x) || !_.isInteger(y)) { + throw new Error(errorPrefix); + } + this.webviewOffset = { + dx: centerX - x, + dy: centerY - y, + }; + } + ); + if (currentUrl) { // restore the previous url await this.setUrl(currentUrl); } + // @ts-ignore webviewOffset is always defined here return {offset: this.webviewOffset}; }, From 0793e05a02c660be50bcd71a8a0c8f012cc80615 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 7 Oct 2023 22:53:41 +0200 Subject: [PATCH 11/21] Address comments --- docs/attach-to-running-wda.md | 5 +++++ lib/commands/web.js | 25 +++++++++++++++---------- lib/driver.js | 4 ++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/docs/attach-to-running-wda.md b/docs/attach-to-running-wda.md index 90a3ac8dc..1410e3535 100644 --- a/docs/attach-to-running-wda.md +++ b/docs/attach-to-running-wda.md @@ -30,3 +30,8 @@ If the environment had port-forward to the connected device, it can be `http://l This method allows you to manage the WebDriverAgent application process by yourself. XCUITest driver simply attaches to the WebDriverAgent application process. It may improve the application performance. + +Some xcuitest driver APIs (for example the [mobile: calibrateWebToRealCoordinatesTranslation](./execute-methods.md#mobile-calibratewebtorealcoordinatestranslation) one) might still require to know +the port number of the remote device if it is a real device. Providing +`webDriverAgentUrl` capability might not be sufficient to recognize the remote port number in case it is different from the local one. Consider settings the `appium:wdaRemotePort` capability value +in such case to supply the driver with the appropriate data. diff --git a/lib/commands/web.js b/lib/commands/web.js index 1d04069ed..2770d5cf1 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -655,11 +655,11 @@ const extensions = { async translateWebCoords(x, y) { this.log.debug(`Translating web coordinates (${JSON.stringify({x, y})}) to native coordinates`); - if (this.webviewOffset) { - this.log.debug(`Will use the previously calibrated offset: ${JSON.stringify(this.webviewOffset)}`); + if (this.calibratedWebviewOffset) { + this.log.debug(`Will use the previously calibrated offset: ${JSON.stringify(this.calibratedWebviewOffset)}`); return { - x: x + this.webviewOffset.dx, - y: y + this.webviewOffset.dy, + x: x + this.calibratedWebviewOffset.dx, + y: y + this.calibratedWebviewOffset.dy, }; } else { this.log.debug( @@ -886,7 +886,7 @@ const extensions = { try { title = await this.title(); this.log.debug(JSON.stringify(title)); - webClickCoordinates = JSON.parse(title); + webClickCoordinates = _.isPlainObject(title) ? title : JSON.parse(title); } catch (e) { throw new Error(`${errorPrefix} Original error: ${e.message}`); } @@ -894,9 +894,14 @@ const extensions = { if (!_.isInteger(x) || !_.isInteger(y)) { throw new Error(errorPrefix); } - this.webviewOffset = { - dx: centerX - x, - dy: centerY - y, + let pixelRatio = 1; + if ((x > centerX) || (y > centerY)) { + // Webview coordinates might be scaled, lets apply a proper ratio there + pixelRatio = /** @type {number} */ (await this.execute('return window.devicePixelRatio;')); + } + this.calibratedWebviewOffset = { + dx: centerX - x / pixelRatio, + dy: centerY - y / pixelRatio, }; } ); @@ -905,8 +910,8 @@ const extensions = { // restore the previous url await this.setUrl(currentUrl); } - // @ts-ignore webviewOffset is always defined here - return {offset: this.webviewOffset}; + // @ts-ignore calibratedWebviewOffset is always defined here + return {offset: this.calibratedWebviewOffset}; }, /** diff --git a/lib/driver.js b/lib/driver.js index 6fdec4602..f82cb2986 100644 --- a/lib/driver.js +++ b/lib/driver.js @@ -202,7 +202,7 @@ class XCUITestDriver extends BaseDriver { curWebFrames; /** @type {import('./types').Delta|null} */ - webviewOffset; + calibratedWebviewOffset; /** * @type {import('./types').Page[]|undefined} @@ -285,7 +285,7 @@ class XCUITestDriver extends BaseDriver { this.webElementsCache = new LRUCache({ max: WEB_ELEMENTS_CACHE_SIZE, }); - this.webviewOffset = null; + this.calibratedWebviewOffset = null; this._waitingAtoms = { count: 0, alertNotifier: new EventEmitter(), From 5077652e6901eb4705f0322eacebac06d99a9462 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 7 Oct 2023 23:08:43 +0200 Subject: [PATCH 12/21] fixes --- docs/execute-methods.md | 9 ++++++++- lib/commands/web.js | 24 +++++++++++++----------- lib/driver.js | 6 +++--- lib/types.ts | 14 +++++++++----- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/docs/execute-methods.md b/docs/execute-methods.md index f5ab1b2a1..d764bb240 100644 --- a/docs/execute-methods.md +++ b/docs/execute-methods.md @@ -1218,7 +1218,14 @@ and may become obsolete. #### Returned Result -An object with a single `offset` property. The property has two subproperties: `dx` and `dy` used to properly shift Safari web element coordinates into native context. +An object with three properties used to properly shift Safari web element coordinates into native context: +- `offsetX`: Webview X offset in real coorrdinates +- `offsetY`: Webview Y offset in real coorrdinates +- `pixelRatio`: Webview pixel ratio + +The following formulas are used for coordinates translation: +`RealX = offsetX + webviewX / pixelRatio` +`RealY = offsetY + webviewY / pixelRatio` ### mobile: updateSafariPreferences diff --git a/lib/commands/web.js b/lib/commands/web.js index 2770d5cf1..89a8674b5 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -655,11 +655,12 @@ const extensions = { async translateWebCoords(x, y) { this.log.debug(`Translating web coordinates (${JSON.stringify({x, y})}) to native coordinates`); - if (this.calibratedWebviewOffset) { - this.log.debug(`Will use the previously calibrated offset: ${JSON.stringify(this.calibratedWebviewOffset)}`); + if (this.webviewCalibrationResult) { + this.log.debug(`Will use the recent calibration result: ${JSON.stringify(this.webviewCalibrationResult)}`); + const { offsetX, offsetY, pixelRatio } = this.webviewCalibrationResult; return { - x: x + this.calibratedWebviewOffset.dx, - y: y + this.calibratedWebviewOffset.dy, + x: offsetX + x / pixelRatio, + y: offsetY + y / pixelRatio, }; } else { this.log.debug( @@ -863,7 +864,7 @@ const extensions = { * to real devices ones in client scripts. * * @this {XCUITestDriver} - * @returns {Promise<{offset: import('../types').Delta}>} + * @returns {Promise} */ async mobileCalibrateWebToRealCoordinatesTranslation() { if (!this.isWebContext()) { @@ -877,7 +878,7 @@ const extensions = { const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?'; await retryInterval( - 10, + 6, 500, async () => { await this.mobileTap(centerX, centerY); @@ -899,9 +900,10 @@ const extensions = { // Webview coordinates might be scaled, lets apply a proper ratio there pixelRatio = /** @type {number} */ (await this.execute('return window.devicePixelRatio;')); } - this.calibratedWebviewOffset = { - dx: centerX - x / pixelRatio, - dy: centerY - y / pixelRatio, + this.webviewCalibrationResult = { + offsetX: centerX - x / pixelRatio, + offsetY: centerY - y / pixelRatio, + pixelRatio, }; } ); @@ -910,8 +912,8 @@ const extensions = { // restore the previous url await this.setUrl(currentUrl); } - // @ts-ignore calibratedWebviewOffset is always defined here - return {offset: this.calibratedWebviewOffset}; + // @ts-ignore it is always defined here + return this.webviewCalibrationResult; }, /** diff --git a/lib/driver.js b/lib/driver.js index f82cb2986..e6cc64469 100644 --- a/lib/driver.js +++ b/lib/driver.js @@ -201,8 +201,8 @@ class XCUITestDriver extends BaseDriver { /** @type {string[]} */ curWebFrames; - /** @type {import('./types').Delta|null} */ - calibratedWebviewOffset; + /** @type {import('./types').CalibrationData|null} */ + webviewCalibrationResult; /** * @type {import('./types').Page[]|undefined} @@ -285,7 +285,7 @@ class XCUITestDriver extends BaseDriver { this.webElementsCache = new LRUCache({ max: WEB_ELEMENTS_CACHE_SIZE, }); - this.calibratedWebviewOffset = null; + this.webviewCalibrationResult = null; this._waitingAtoms = { count: 0, alertNotifier: new EventEmitter(), diff --git a/lib/types.ts b/lib/types.ts index ccbfd62e3..11721dc3a 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -66,13 +66,17 @@ export interface WDACapabilities { capabilities?: StringRecord; } -export interface Delta { +export interface CalibrationData { /** - * x offset + * webview x offset in real coordinates */ - dx: number; + offsetX: number; /** - * y offset + * webview y offset in real coordinates */ - dy: number; + offsetY: number; + /** + * pixel ratio inside of the web view + */ + pixelRatio: number; } From 8688b396ed6bd0a660265884b0b2eb5fada0b36d Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 7 Oct 2023 23:09:22 +0200 Subject: [PATCH 13/21] Fix typo --- docs/execute-methods.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/execute-methods.md b/docs/execute-methods.md index d764bb240..84e338977 100644 --- a/docs/execute-methods.md +++ b/docs/execute-methods.md @@ -1219,8 +1219,8 @@ and may become obsolete. #### Returned Result An object with three properties used to properly shift Safari web element coordinates into native context: -- `offsetX`: Webview X offset in real coorrdinates -- `offsetY`: Webview Y offset in real coorrdinates +- `offsetX`: Webview X offset in real coordinates +- `offsetY`: Webview Y offset in real coordinates - `pixelRatio`: Webview pixel ratio The following formulas are used for coordinates translation: From e0661e6822a17703c4f608f0047755b145af94e3 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sat, 7 Oct 2023 23:10:54 +0200 Subject: [PATCH 14/21] Use trunc --- lib/commands/web.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 89a8674b5..0008cc8fa 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -659,8 +659,8 @@ const extensions = { this.log.debug(`Will use the recent calibration result: ${JSON.stringify(this.webviewCalibrationResult)}`); const { offsetX, offsetY, pixelRatio } = this.webviewCalibrationResult; return { - x: offsetX + x / pixelRatio, - y: offsetY + y / pixelRatio, + x: Math.trunc(offsetX + x / pixelRatio), + y: Math.trunc(offsetY + y / pixelRatio), }; } else { this.log.debug( From afd4b1859793da2215b63e3092e21db00c5daf5e Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sun, 8 Oct 2023 19:49:50 +0200 Subject: [PATCH 15/21] Use native pixel ratio --- lib/commands/web.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 0008cc8fa..23fcb9232 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -895,11 +895,7 @@ const extensions = { if (!_.isInteger(x) || !_.isInteger(y)) { throw new Error(errorPrefix); } - let pixelRatio = 1; - if ((x > centerX) || (y > centerY)) { - // Webview coordinates might be scaled, lets apply a proper ratio there - pixelRatio = /** @type {number} */ (await this.execute('return window.devicePixelRatio;')); - } + const pixelRatio = (x > centerX || y > centerY) ? (await this.getDevicePixelRatio()) : 1; this.webviewCalibrationResult = { offsetX: centerX - x / pixelRatio, offsetY: centerY - y / pixelRatio, From 6e9a40532e9c3dd4d6d80d514a874118a3560c73 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Sun, 8 Oct 2023 19:50:45 +0200 Subject: [PATCH 16/21] Apply truncation --- lib/commands/web.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 23fcb9232..028f22352 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -897,8 +897,8 @@ const extensions = { } const pixelRatio = (x > centerX || y > centerY) ? (await this.getDevicePixelRatio()) : 1; this.webviewCalibrationResult = { - offsetX: centerX - x / pixelRatio, - offsetY: centerY - y / pixelRatio, + offsetX: Math.trunc(centerX - x / pixelRatio), + offsetY: Math.trunc(centerY - y / pixelRatio), pixelRatio, }; } From 6a73b0485ec7d414d47166be1138db2ecca5caf9 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Mon, 9 Oct 2023 14:58:48 +0200 Subject: [PATCH 17/21] Perform calibration with delta --- docs/execute-methods.md | 7 +++--- lib/commands/web.js | 56 +++++++++++++++++++++++++---------------- lib/types.ts | 8 ++++-- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/docs/execute-methods.md b/docs/execute-methods.md index 84e338977..3077df932 100644 --- a/docs/execute-methods.md +++ b/docs/execute-methods.md @@ -1221,11 +1221,12 @@ and may become obsolete. An object with three properties used to properly shift Safari web element coordinates into native context: - `offsetX`: Webview X offset in real coordinates - `offsetY`: Webview Y offset in real coordinates -- `pixelRatio`: Webview pixel ratio +- `pixelRatioX`: Webview X pixel ratio +- `pixelRatioY`: Webview Y pixel ratio The following formulas are used for coordinates translation: -`RealX = offsetX + webviewX / pixelRatio` -`RealY = offsetY + webviewY / pixelRatio` +`RealX = offsetX + webviewX * pixelRatioX` +`RealY = offsetY + webviewY * pixelRatioY` ### mobile: updateSafariPreferences diff --git a/lib/commands/web.js b/lib/commands/web.js index 028f22352..28b31c771 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -17,6 +17,8 @@ const TAB_BAR_OFFSET = 33; const IPHONE_WEB_COORD_SMART_APP_BANNER_OFFSET = 84; const IPAD_WEB_COORD_SMART_APP_BANNER_OFFSET = 95; +const CALIBRATION_TAP_DELTA = 5; // px + const NOTCHED_DEVICE_SIZES = [ {w: 1125, h: 2436}, // 11 Pro, X, Xs {w: 828, h: 1792}, // 11, Xr @@ -657,10 +659,10 @@ const extensions = { if (this.webviewCalibrationResult) { this.log.debug(`Will use the recent calibration result: ${JSON.stringify(this.webviewCalibrationResult)}`); - const { offsetX, offsetY, pixelRatio } = this.webviewCalibrationResult; + const { offsetX, offsetY, pixelRatioX, pixelRatioY } = this.webviewCalibrationResult; return { - x: Math.trunc(offsetX + x / pixelRatio), - y: Math.trunc(offsetY + y / pixelRatio), + x: Math.trunc(offsetX + x * pixelRatioX), + y: Math.trunc(offsetY + y * pixelRatioY), }; } else { this.log.debug( @@ -877,29 +879,41 @@ const extensions = { const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?'; + const performCalibrationTap = async (/** @type {number} */ tapX, /** @type {number} */ tapY) => { + await this.mobileTap(tapX, tapY); + /** @type {import('@appium/types').Position} */ + let result; + try { + const title = await this.title(); + this.log.debug(JSON.stringify(title)); + result = _.isPlainObject(title) ? title : JSON.parse(title); + } catch (e) { + throw new Error(`${errorPrefix} Original error: ${e.message}`); + } + const {x, y} = result; + if (!_.isInteger(x) || !_.isInteger(y)) { + throw new Error(errorPrefix); + } + return result; + }; + await retryInterval( 6, 500, async () => { - await this.mobileTap(centerX, centerY); - let webClickCoordinates; - let title; - try { - title = await this.title(); - this.log.debug(JSON.stringify(title)); - webClickCoordinates = _.isPlainObject(title) ? title : JSON.parse(title); - } catch (e) { - throw new Error(`${errorPrefix} Original error: ${e.message}`); - } - const {x, y} = webClickCoordinates; - if (!_.isInteger(x) || !_.isInteger(y)) { - throw new Error(errorPrefix); - } - const pixelRatio = (x > centerX || y > centerY) ? (await this.getDevicePixelRatio()) : 1; + const {x: x0, y: y0} = await performCalibrationTap( + centerX - CALIBRATION_TAP_DELTA, centerY - CALIBRATION_TAP_DELTA + ); + const {x: x1, y: y1} = await performCalibrationTap( + centerX + CALIBRATION_TAP_DELTA, centerY + CALIBRATION_TAP_DELTA + ); + const pixelRatioX = CALIBRATION_TAP_DELTA * 2 / (x1 - x0); + const pixelRatioY = CALIBRATION_TAP_DELTA * 2 / (y1 - y0); this.webviewCalibrationResult = { - offsetX: Math.trunc(centerX - x / pixelRatio), - offsetY: Math.trunc(centerY - y / pixelRatio), - pixelRatio, + offsetX: Math.trunc(centerX - CALIBRATION_TAP_DELTA - x0 * pixelRatioX), + offsetY: Math.trunc(centerY - CALIBRATION_TAP_DELTA - y0 * pixelRatioY), + pixelRatioX, + pixelRatioY, }; } ); diff --git a/lib/types.ts b/lib/types.ts index 11721dc3a..156603b04 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -76,7 +76,11 @@ export interface CalibrationData { */ offsetY: number; /** - * pixel ratio inside of the web view + * pixel ratio x inside of the web view */ - pixelRatio: number; + pixelRatioX: number; + /** + * pixel ratio y inside of the web view + */ + pixelRatioY: number; } From 59699c54810650bc5e2b0a6d967edfdb2794ba4d Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Tue, 10 Oct 2023 08:25:44 +0200 Subject: [PATCH 18/21] Use round --- lib/commands/web.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 28b31c771..9f7f1d4fe 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -661,8 +661,8 @@ const extensions = { this.log.debug(`Will use the recent calibration result: ${JSON.stringify(this.webviewCalibrationResult)}`); const { offsetX, offsetY, pixelRatioX, pixelRatioY } = this.webviewCalibrationResult; return { - x: Math.trunc(offsetX + x * pixelRatioX), - y: Math.trunc(offsetY + y * pixelRatioY), + x: Math.round(offsetX + x * pixelRatioX), + y: Math.round(offsetY + y * pixelRatioY), }; } else { this.log.debug( @@ -876,7 +876,7 @@ const extensions = { const currentUrl = await this.getUrl(); await this.setUrl(`${this.getWdaLocalhostRoot()}/calibrate`); const {width, height} = /** @type {import('@appium/types').Size} */(await this.proxyCommand(`/window/size`, 'GET')); - const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)]; + const [centerX, centerY] = [width / 2, height / 2]; const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?'; const performCalibrationTap = async (/** @type {number} */ tapX, /** @type {number} */ tapY) => { @@ -910,8 +910,8 @@ const extensions = { const pixelRatioX = CALIBRATION_TAP_DELTA * 2 / (x1 - x0); const pixelRatioY = CALIBRATION_TAP_DELTA * 2 / (y1 - y0); this.webviewCalibrationResult = { - offsetX: Math.trunc(centerX - CALIBRATION_TAP_DELTA - x0 * pixelRatioX), - offsetY: Math.trunc(centerY - CALIBRATION_TAP_DELTA - y0 * pixelRatioY), + offsetX: Math.round(centerX - CALIBRATION_TAP_DELTA - x0 * pixelRatioX), + offsetY: Math.round(centerY - CALIBRATION_TAP_DELTA - y0 * pixelRatioY), pixelRatioX, pixelRatioY, }; From 369d8b4bf0d533d93b90042ecefc76b6d40f25fe Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Wed, 11 Oct 2023 18:12:59 +0200 Subject: [PATCH 19/21] Decide whether to apply scale --- lib/commands/web.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 9f7f1d4fe..221953ac6 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -660,9 +660,15 @@ const extensions = { if (this.webviewCalibrationResult) { this.log.debug(`Will use the recent calibration result: ${JSON.stringify(this.webviewCalibrationResult)}`); const { offsetX, offsetY, pixelRatioX, pixelRatioY } = this.webviewCalibrationResult; + const cmd = '(function () {return {innerWidth: window.innerWidth, innerHeight: window.innerHeight, ' + + 'outerWidth: window.outerWidth, outerHeight: window.outerHeight}; })()'; + const wvDims = await this.remote.execute(cmd); + // https://tripleodeon.com/2011/12/first-understand-your-screen/ + const shouldApplyPixelRatio = wvDims.innerWidth > wvDims.outerWidth + || wvDims.innerHeight > wvDims.outerHeight; return { - x: Math.round(offsetX + x * pixelRatioX), - y: Math.round(offsetY + y * pixelRatioY), + x: Math.round(offsetX + x * (shouldApplyPixelRatio ? pixelRatioX : 1)), + y: Math.round(offsetY + y * (shouldApplyPixelRatio ? pixelRatioY : 1)), }; } else { this.log.debug( From 9c3e5b3e1b39992b9b66dd4140aa6936f3247f1e Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Wed, 11 Oct 2023 18:19:57 +0200 Subject: [PATCH 20/21] Leave it float --- lib/commands/web.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/commands/web.js b/lib/commands/web.js index 221953ac6..0477c19e6 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -667,8 +667,8 @@ const extensions = { const shouldApplyPixelRatio = wvDims.innerWidth > wvDims.outerWidth || wvDims.innerHeight > wvDims.outerHeight; return { - x: Math.round(offsetX + x * (shouldApplyPixelRatio ? pixelRatioX : 1)), - y: Math.round(offsetY + y * (shouldApplyPixelRatio ? pixelRatioY : 1)), + x: offsetX + x * (shouldApplyPixelRatio ? pixelRatioX : 1), + y: offsetY + y * (shouldApplyPixelRatio ? pixelRatioY : 1), }; } else { this.log.debug( @@ -916,8 +916,8 @@ const extensions = { const pixelRatioX = CALIBRATION_TAP_DELTA * 2 / (x1 - x0); const pixelRatioY = CALIBRATION_TAP_DELTA * 2 / (y1 - y0); this.webviewCalibrationResult = { - offsetX: Math.round(centerX - CALIBRATION_TAP_DELTA - x0 * pixelRatioX), - offsetY: Math.round(centerY - CALIBRATION_TAP_DELTA - y0 * pixelRatioY), + offsetX: centerX - CALIBRATION_TAP_DELTA - x0 * pixelRatioX, + offsetY: centerY - CALIBRATION_TAP_DELTA - y0 * pixelRatioY, pixelRatioX, pixelRatioY, }; @@ -928,8 +928,14 @@ const extensions = { // restore the previous url await this.setUrl(currentUrl); } - // @ts-ignore it is always defined here - return this.webviewCalibrationResult; + /** @type {import('../types').CalibrationData} */ + // @ts-ignore this.webviewCalibrationResult is always defined here + const result = this.webviewCalibrationResult; + return { + ...result, + offsetX: Math.round(result.offsetX), + offsetY: Math.round(result.offsetY), + }; }, /** From 1de200ac14b87e173c6b1a8cb48743b8c03d386c Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Fri, 13 Oct 2023 08:34:54 +0200 Subject: [PATCH 21/21] Address comments --- docs/capabilities.md | 2 +- docs/execute-methods.md | 7 ++- lib/commands/web.js | 108 +++++++++++++++++++++++----------------- 3 files changed, 67 insertions(+), 50 deletions(-) diff --git a/docs/capabilities.md b/docs/capabilities.md index f3f25559b..edb6b3bc0 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -123,7 +123,7 @@ Capability | Description |`appium:webviewConnectTimeout`|The time to wait, in `ms`, for the initial presence of webviews in MobileSafari or hybrid apps. Defaults to `0`|e.g., '5000'| |`appium:safariIgnoreWebHostnames`| Provide a list of hostnames (comma-separated) that the Safari automation tools should ignore. This is to provide a workaround to prevent a webkit bug where the web context is unintentionally changed to a 3rd party website and the test gets stuck. The common culprits are search engines (yahoo, bing, google) and `about:blank` |e.g. `'www.yahoo.com, www.bing.com, www.google.com, about:blank'`| |`appium:nativeWebTap`| Enable native, non-javascript-based taps being in web context mode. Defaults to `false`. Warning: sometimes the preciseness of native taps could be broken, because there is no reliable way to map web element coordinates to native ones. | `true` | -|`appium:nativeWebTapStrict`| Enforce native taps to be done by XCUITest driver rather than WebDriverAgent. Only applicable if `nativeWebTap` is enabled. `false` by default | `false` | +|`appium:nativeWebTapStrict`| Enabling this capability would skip the additional logic that tries to match web view elements to native ones by using their textual descriptions. Depending on the actual web view content this algorithm might sometimes be not very reliable and will slow down each click as we anyway fallback to the usual coordinates transformation flow if it fails. It is advised to enable strict tap if you use [mobile: calibrateWebToRealCoordinatesTranslation extension](./execute-methods.md#mobile-calibratewebtorealcoordinatestranslation). Only applicable if `nativeWebTap` is enabled. `false` by default | `true` | |`appium:safariInitialUrl`| Initial safari url, default is a local welcome page. Setting it to an empty string will skip the initial navigation. | e.g. `https://www.github.com` | |`appium:safariAllowPopups`| (Simulator only) Allow javascript to open new windows in Safari. Default keeps current sim setting. |`true` or `false`| |`appium:safariIgnoreFraudWarning`| (Simulator only) Prevent Safari from showing a fraudulent website warning. Default keeps current sim setting. |`true` or `false`| diff --git a/docs/execute-methods.md b/docs/execute-methods.md index 3077df932..308b74998 100644 --- a/docs/execute-methods.md +++ b/docs/execute-methods.md @@ -1208,14 +1208,17 @@ This API can only be called from Safari web context. It must load a custom page to the browser, and then restore the original one, so don't call it if you can potentially lose the current web app state. -The outcome of this API is then used in `nativeWebTap` mode. +The outcome of this API is then used if `nativeWebTap` capability/setting is enabled. The returned value could also be used to manually transform web coordinates -to real devices ones in client scripts. +to real device ones in client scripts. It is adviced to call this API at least once before changing the device orientation or device screen layout as the recetly received value is cached for the session lifetime and may become obsolete. +It is advised to enable `nativeWebTapStrict` capability/setting to speed up dynamic coordinates +transformation if you use this extension. + #### Returned Result An object with three properties used to properly shift Safari web element coordinates into native context: diff --git a/lib/commands/web.js b/lib/commands/web.js index 0477c19e6..9a3fc7f85 100644 --- a/lib/commands/web.js +++ b/lib/commands/web.js @@ -17,7 +17,7 @@ const TAB_BAR_OFFSET = 33; const IPHONE_WEB_COORD_SMART_APP_BANNER_OFFSET = 84; const IPAD_WEB_COORD_SMART_APP_BANNER_OFFSET = 95; -const CALIBRATION_TAP_DELTA = 5; // px +const CALIBRATION_TAP_DELTA_PX = 7; const NOTCHED_DEVICE_SIZES = [ {w: 1125, h: 2436}, // 11 Pro, X, Xs @@ -52,50 +52,62 @@ const TAB_BAR_POSITION_TOP = 'top'; const TAB_BAR_POSITION_BOTTOM = 'bottom'; const TAB_BAR_POSSITIONS = [TAB_BAR_POSITION_TOP, TAB_BAR_POSITION_BOTTOM]; -async function tapWebElementNatively(driver, atomsElement) { +/** + * @this {XCUITestDriver} + * @param {any} atomsElement + * @returns {Promise} + */ +async function tapWebElementNatively(atomsElement) { // try to get the text of the element, which will be accessible in the // native context try { - let text = await driver.executeAtom('get_text', [atomsElement]); + const [text1, text2] = await B.all([ + this.executeAtom('get_text', [atomsElement]), + this.executeAtom('get_attribute_value', [atomsElement, 'value']) + ]); + const text = text1 || text2; if (!text) { - text = await driver.executeAtom('get_attribute_value', [atomsElement, 'value']); - } - - if (text) { - const els = await driver.findNativeElementOrElements('accessibility id', text, true); - if (els.length === 1 || els.length === 2) { - const el = els[0]; - // use tap because on iOS 11.2 and below `nativeClick` crashes WDA - const rect = await driver.proxyCommand(`/element/${util.unwrapElement(el)}/rect`, 'GET'); - if (els.length === 2) { - const el2 = els[1]; - const rect2 = await driver.proxyCommand( - `/element/${util.unwrapElement(el2)}/rect`, - 'GET', - ); - - if ( - rect.x !== rect2.x || - rect.y !== rect2.y || - rect.width !== rect2.width || - rect.height !== rect2.height - ) { - // These 2 native elements are not referring to the same web element - return false; - } - } - await driver.mobileTap(Math.round(rect.x + rect.width / 2), Math.round(rect.y + rect.height / 2)); - return true; + return false; + } + + const els = await this.findNativeElementOrElements('accessibility id', text, true); + if (![1, 2].includes(els.length)) { + return false; + } + + const el = els[0]; + // use tap because on iOS 11.2 and below `nativeClick` crashes WDA + const rect = /** @type {import('@appium/types').Rect} */ (await this.proxyCommand( + `/element/${util.unwrapElement(el)}/rect`, 'GET' + )); + if (els.length > 1) { + const el2 = els[1]; + const rect2 = /** @type {import('@appium/types').Rect} */ (await this.proxyCommand( + `/element/${util.unwrapElement(el2)}/rect`, 'GET', + )); + + if ( + rect.x !== rect2.x || rect.y !== rect2.y + || rect.width !== rect2.width || rect.height !== rect2.height + ) { + // These 2 native elements are not referring to the same web element + return false; } } + await this.mobileTap(rect.x + rect.width / 2, rect.y + rect.height / 2); + return true; } catch (err) { // any failure should fall through and trigger the more elaborate // method of clicking - driver.log.warn(`Error attempting to click: ${err.message}`); + this.log.warn(`Error attempting to click: ${err.message}`); } return false; } +/** + * @param {any} id + * @returns {boolean} + */ function isValidElementIdentifier(id) { if (!_.isString(id) && !_.isNumber(id)) { return false; @@ -442,6 +454,7 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @returns {Promise} */ async getSafariIsIphone() { if (_.isBoolean(this._isSafariIphone)) { @@ -458,6 +471,7 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @returns {Promise} */ async getSafariDeviceSize() { const script = @@ -473,6 +487,7 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @returns {Promise} */ async getSafariIsNotched() { if (_.isBoolean(this._isSafariNotched)) { @@ -592,6 +607,9 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @param {boolean} isIphone + * @param {string} bannerVisibility + * @returns {Promise} */ async getExtraNativeWebTapOffset(isIphone, bannerVisibility) { let offset = 0; @@ -617,26 +635,21 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @param {any} el + * @returns {Promise} */ async nativeWebTap(el) { const atomsElement = this.getAtomsElement(el); // if strict native tap, do not try to do it with WDA directly if ( - !(await this.settings.getSettings()).nativeWebTapStrict && - (await tapWebElementNatively(this, atomsElement)) + !(this.settings.getSettings()).nativeWebTapStrict && + (await tapWebElementNatively.bind(this)(atomsElement)) ) { return; } this.log.warn('Unable to do simple native web tap. Attempting to convert coordinates'); - // `get_top_left_coordinates` returns the wrong value sometimes, - // unless we pre-call both of these functions before the actual calls - await B.Promise.all([ - this.executeAtom('get_size', [atomsElement]), - this.executeAtom('get_top_left_coordinates', [atomsElement]), - ]); - const [size, coordinates] = /** @type {[import('@appium/types').Size, import('@appium/types').Position]} */ ( await B.Promise.all([ @@ -747,6 +760,7 @@ const extensions = { }, /** * @this {XCUITestDriver} + * @returns {Promise} */ async checkForAlert() { return _.isString(await this.getAlertText()); @@ -908,16 +922,16 @@ const extensions = { 500, async () => { const {x: x0, y: y0} = await performCalibrationTap( - centerX - CALIBRATION_TAP_DELTA, centerY - CALIBRATION_TAP_DELTA + centerX - CALIBRATION_TAP_DELTA_PX, centerY - CALIBRATION_TAP_DELTA_PX ); const {x: x1, y: y1} = await performCalibrationTap( - centerX + CALIBRATION_TAP_DELTA, centerY + CALIBRATION_TAP_DELTA + centerX + CALIBRATION_TAP_DELTA_PX, centerY + CALIBRATION_TAP_DELTA_PX ); - const pixelRatioX = CALIBRATION_TAP_DELTA * 2 / (x1 - x0); - const pixelRatioY = CALIBRATION_TAP_DELTA * 2 / (y1 - y0); + const pixelRatioX = CALIBRATION_TAP_DELTA_PX * 2 / (x1 - x0); + const pixelRatioY = CALIBRATION_TAP_DELTA_PX * 2 / (y1 - y0); this.webviewCalibrationResult = { - offsetX: centerX - CALIBRATION_TAP_DELTA - x0 * pixelRatioX, - offsetY: centerY - CALIBRATION_TAP_DELTA - y0 * pixelRatioY, + offsetX: centerX - CALIBRATION_TAP_DELTA_PX - x0 * pixelRatioX, + offsetY: centerY - CALIBRATION_TAP_DELTA_PX - y0 * pixelRatioY, pixelRatioX, pixelRatioY, };