feat: Add 'mobile: calibrateWebToRealCoordinatesTranslation' API

docs/attach-to-running-wda.md

@@ -30,3 +30,8 @@ If the environment had port-forward to the connected device, it can be `http://l
 This method allows you to manage the WebDriverAgent application process by yourself.
 XCUITest driver simply attaches to the WebDriverAgent application process.
 It may improve the application performance.
+
+Some xcuitest driver APIs (for example the [mobile: calibrateWebToRealCoordinatesTranslation](./execute-methods.md#mobile-calibratewebtorealcoordinatestranslation) one) might still require to know
+the port number of the remote device if it is a real device. Providing
+`webDriverAgentUrl` capability might not be sufficient to recognize the remote port number in case it is different from the local one. Consider settings the `appium:wdaRemotePort` capability value
+in such case to supply the driver with the appropriate data.

docs/capabilities.md

@@ -52,6 +52,7 @@ Capability | Description
 |`appium:wdaStartupRetries`|Number of times to try to build and launch `WebDriverAgent` onto the device. Defaults to 2.|e.g., `4`|
 |`appium:wdaStartupRetryInterval`|Time, in ms, to wait between tries to build and launch `WebDriverAgent`. Defaults to 10000ms.|e.g., `20000`|
 |`appium:wdaLocalPort`|This value if specified, will be used to forward traffic from Mac host to real ios devices over USB. Default value is same as port number used by WDA on device.|e.g., `8100`|
+|`appium:wdaRemotePort`|This value if specified, will be used as the port number to start WDA HTTP server on the remote device. This is only relevant for real devices, because Simulator shares ports with its host. If `webDriverAgentUrl` is provided then it might be used to provide a hint for the remote port number if it differs from the default one. Default value is 8100.|e.g., `8100`|
 |`appium:wdaBaseUrl`| This value if specified, will be used as a prefix to build a custom `WebDriverAgent` url. It is different from `webDriverAgentUrl`, because if the latter is set then it expects `WebDriverAgent` to be already listening and skips the building phase. Defaults to `http://localhost` | e.g., `http://192.168.1.100`|
 |`appium:showXcodeLog`|Whether to display the output of the Xcode command used to run the tests. If this is `true`, there will be **lots** of extra logging at startup. Defaults to `false`|e.g., `true`|
 |`appium:iosInstallPause`|Time in milliseconds to pause between installing the application and starting `WebDriverAgent` on the device. Used particularly for larger applications. Defaults to `0`|e.g., `8000`|

docs/endpoints.md

@@ -65,7 +65,6 @@
 | POST   | /alert_text                              | text | |
 | POST   | /accept_alert                            | none | |
 | POST   | /dismiss_alert                           | none | |
-| POST   | /moveto                                  | | element, xoffset, yoffset |
 | POST   | /click                                   | | button |
 | POST   | /touch/click                             | element | |
 | POST   | /touch/flick                             | | element, xspeed, yspeed, xoffset, yoffset, speed |

docs/execute-methods.md

@@ -1201,6 +1201,32 @@ If the connection is disconnected, condition inducer will be automatically disab
 
 Either `true` or `false`, where `true` means disabling of the condition inducer has been successful
 
+### mobile: calibrateWebToRealCoordinatesTranslation
+
+Calibrates web to real coordinates translation.
+This API can only be called from Safari web context.
+It must load a custom page to the browser, and then restore
+the original one, so don't call it if you can potentially
+lose the current web app state.
+The outcome of this API is then used in `nativeWebTap` mode.
+The returned value could also be used to manually transform web coordinates
+to real devices ones in client scripts.
+
+It is adviced to call this API at least once before changing the device orientation
+or device screen layout as the recetly received value is cached for the session lifetime
+and may become obsolete.
+
+#### Returned Result
+
+An object with three properties used to properly shift Safari web element coordinates into native context:
+- `offsetX`: Webview X offset in real coordinates
+- `offsetY`: Webview Y offset in real coordinates
+- `pixelRatio`: Webview pixel ratio
+
+The following formulas are used for coordinates translation:
+`RealX = offsetX + webviewX / pixelRatio`
+`RealY = offsetY + webviewY / pixelRatio`
+
 ### mobile: updateSafariPreferences
 
 Updates preferences of Mobile Safari on Simulator

lib/commands/context.js

@@ -392,7 +392,6 @@ const helpers = {
     await this.remote.disconnect();
     this.curContext = null;
     this.curWebFrames = [];
-    this.curWebCoords = null;
     this.remote = null;
   },
   /**

lib/commands/gesture.js

@@ -97,43 +97,6 @@ export function gesturesChainToString(gestures, keysToInclude = ['options']) {
 }
 
 const commands = {
-  /**
-   * Move the mouse pointer to a particular screen location
-   *
-   * @param {string|Element} el - the element ID if the move is relative to an element
-   * @param {number} xoffset - the x offset
-   * @param {number} yoffset - the y offset
-   * @this {XCUITestDriver}
-   * @deprecated Use {@linkcode XCUITestDriver.performActions} instead
-   */
-  async moveTo(el, xoffset = 0, yoffset = 0) {
-    el = util.unwrapElement(el);
-
-    if (this.isWebContext()) {
-      throw new errors.UnknownMethodError(
-        'The moveTo command is not available in the web context. Use the Actions API in the ' +
-          'native context instead',
-      );
-    } else {
-      if (_.isNil(el)) {
-        if (!this.curCoords) {
-          throw new errors.UnknownError(
-            'Current cursor position unknown, please use moveTo with an element the first time.',
-          );
-        }
-        this.curCoords = {
-          x: this.curCoords.x + xoffset,
-          y: this.curCoords.y + yoffset,
-        };
-      } else {
-        let elPos = await this.getLocation(el);
-        this.curCoords = {
-          x: elPos.x + xoffset,
-          y: elPos.y + yoffset,
-        };
-      }
-    }
-  },
   /**
    * Shake the device
    * @this {XCUITestDriver}
@@ -767,19 +730,6 @@ const helpers = {
     }
     return coordinates;
   },
-  /**
-   * @this {XCUITestDriver}
-   */
-  applyMoveToOffset(firstCoordinates, secondCoordinates) {
-    if (secondCoordinates.areOffsets) {
-      return {
-        x: firstCoordinates.x + secondCoordinates.x,
-        y: firstCoordinates.y + secondCoordinates.y,
-      };
-    } else {
-      return secondCoordinates;
-    }
-  },
 };
 
 export default {...helpers, ...commands};

lib/commands/proxy-helper.js

@@ -49,9 +49,6 @@ const WDA_ROUTES = /** @type {const} */ ({
   '/wda/locked': {
     GET: 'isLocked',
   },
-  '/wda/tap/nil': {
-    POST: 'clickCoords',
-  },
   '/window/size': {
     GET: 'getWindowSize',
   },

lib/commands/web.js

@@ -82,11 +82,7 @@ async function tapWebElementNatively(driver, atomsElement) {
             return false;
           }
         }
-        const coords = {
-          x: Math.round(rect.x + rect.width / 2),
-          y: Math.round(rect.y + rect.height / 2),
-        };
-        await driver.clickCoords(coords);
+        await driver.mobileTap(Math.round(rect.x + rect.width / 2), Math.round(rect.y + rect.height / 2));
         return true;
       }
     }
@@ -435,10 +431,12 @@ const extensions = {
   },
   /**
    * @this {XCUITestDriver}
+   * @param {number} x
+   * @param {number} y
    */
-  async clickWebCoords() {
-    let coords = await this.translateWebCoords(this.curWebCoords);
-    await this.clickCoords(coords);
+  async clickWebCoords(x, y) {
+    const {x: translatedX, y: translatedY} = await this.translateWebCoords(x, y);
+    await this.mobileTap(translatedX, translatedY);
   },
   /**
    * @this {XCUITestDriver}
@@ -645,29 +643,31 @@ const extensions = {
         ])
       );
     const {width, height} = size;
-    let {x, y} = coordinates;
-    x += width / 2;
-    y += height / 2;
-
-    this.curWebCoords = {x, y};
-    await this.clickWebCoords();
-  },
-  /**
-   * @this {XCUITestDriver}
-   */
-  async clickCoords(coords) {
-    await this.performTouch([
-      {
-        action: 'tap',
-        options: coords,
-      },
-    ]);
+    const {x, y} = coordinates;
+    await this.clickWebCoords(x + width / 2, y + height / 2);
   },
   /**
    * @this {XCUITestDriver}
-   */
-  async translateWebCoords(coords) {
-    this.log.debug(`Translating coordinates (${JSON.stringify(coords)}) to web coordinates`);
+   * @param {number} x
+   * @param {number} y
+   * @returns {Promise<import('@appium/types').Position>}
+   */
+  async translateWebCoords(x, y) {
+    this.log.debug(`Translating web coordinates (${JSON.stringify({x, y})}) to native coordinates`);
+
+    if (this.webviewCalibrationResult) {
+      this.log.debug(`Will use the recent calibration result: ${JSON.stringify(this.webviewCalibrationResult)}`);
+      const { offsetX, offsetY, pixelRatio } = this.webviewCalibrationResult;
+      return {
+        x: Math.trunc(offsetX + x / pixelRatio),
+        y: Math.trunc(offsetY + y / pixelRatio),
+      };
+    } else {
+      this.log.debug(
+        `Using the legacy algorithm for coordinates translation. ` +
+        `Invoke 'mobile: calibrateWebToRealCoordinatesTranslation' to change that.`
+      );
+    }
 
     // absolutize web coords
     /** @type {import('@appium/types').Element|undefined|string} */
@@ -705,32 +705,37 @@ const extensions = {
     } finally {
       this.setImplicitWait(implicitWaitMs);
     }
-
-    if (wvDims && realDims && wvPos) {
-      let xRatio = realDims.w / wvDims.w;
-      let yRatio = realDims.h / wvDims.h;
-      let newCoords = {
-        x: wvPos.x + Math.round(xRatio * coords.x),
-        y: wvPos.y + Math.round(yRatio * coords.y),
-      };
-
-      // additional logging for coordinates, since it is sometimes broken
-      //   see https://github.com/appium/appium/issues/9159
-      this.log.debug(`Converted coordinates: ${JSON.stringify(newCoords)}`);
-      this.log.debug(`    rect: ${JSON.stringify(rect)}`);
-      this.log.debug(`    wvPos: ${JSON.stringify(wvPos)}`);
-      this.log.debug(`    realDims: ${JSON.stringify(realDims)}`);
-      this.log.debug(`    wvDims: ${JSON.stringify(wvDims)}`);
-      this.log.debug(`    xRatio: ${JSON.stringify(xRatio)}`);
-      this.log.debug(`    yRatio: ${JSON.stringify(yRatio)}`);
-
-      this.log.debug(
-        `Converted web coords ${JSON.stringify(coords)} into real coords ${JSON.stringify(
-          newCoords,
-        )}`,
+    if (!wvDims || !realDims || !wvPos) {
+      throw new Error(
+        `Web coordinates ${JSON.stringify({x, y})} cannot be translated into real coordinates. ` +
+        `Try to invoke 'mobile: calibrateWebToRealCoordinatesTranslation' or consider translating the ` +
+        `coordinates from the client code.`
       );
-      return newCoords;
     }
+
+    const xRatio = realDims.w / wvDims.w;
+    const yRatio = realDims.h / wvDims.h;
+    const newCoords = {
+      x: wvPos.x + Math.round(xRatio * x),
+      y: wvPos.y + Math.round(yRatio * y),
+    };
+
+    // additional logging for coordinates, since it is sometimes broken
+    //   see https://github.com/appium/appium/issues/9159
+    this.log.debug(`Converted coordinates: ${JSON.stringify(newCoords)}`);
+    this.log.debug(`    rect: ${JSON.stringify(rect)}`);
+    this.log.debug(`    wvPos: ${JSON.stringify(wvPos)}`);
+    this.log.debug(`    realDims: ${JSON.stringify(realDims)}`);
+    this.log.debug(`    wvDims: ${JSON.stringify(wvDims)}`);
+    this.log.debug(`    xRatio: ${JSON.stringify(xRatio)}`);
+    this.log.debug(`    yRatio: ${JSON.stringify(yRatio)}`);
+
+    this.log.debug(
+      `Converted web coords ${JSON.stringify({x, y})} into real coords ${JSON.stringify(
+        newCoords,
+      )}`,
+    );
+    return newCoords;
   },
   /**
    * @this {XCUITestDriver}
@@ -834,6 +839,83 @@ const extensions = {
     }
   },
 
+  /**
+   * @this {XCUITestDriver}
+   * @returns {string} The base url which could be used to access WDA HTTP endpoints
+   * FROM THE SAME DEVICE where WDA is running
+   */
+  getWdaLocalhostRoot() {
+    const remotePort =
+      (this.opts.wdaRemotePort
+      ?? this.wda?.url?.port
+      ?? this.opts.wdaLocalPort)
+      || 8100;
+    return `http://127.0.0.1:${remotePort}`;
+  },
+
+  /**
+   * Calibrates web to real coordinates translation.
+   * This API can only be called from Safari web context.
+   * It must load a custom page to the browser, and then restore
+   * the original one, so don't call it if you can potentially
+   * lose the current web app state.
+   * The outcome of this API is then used in nativeWebTap mode.
+   * The returned value could also be used to manually transform web coordinates
+   * to real devices ones in client scripts.
+   *
+   * @this {XCUITestDriver}
+   * @returns {Promise<import('../types').CalibrationData>}
+   */
+  async mobileCalibrateWebToRealCoordinatesTranslation() {
+    if (!this.isWebContext()) {
+      throw new errors.NotImplementedError('This API can only be called from a web context');
+    }
+
+    const currentUrl = await this.getUrl();
+    await this.setUrl(`${this.getWdaLocalhostRoot()}/calibrate`);
+    const {width, height} = /** @type {import('@appium/types').Size} */(await this.proxyCommand(`/window/size`, 'GET'));
+    const [centerX, centerY] = [Math.trunc(width / 2), Math.trunc(height / 2)];
+    const errorPrefix = 'Cannot determine web view coordinates offset. Are you in Safari context?';
+
+    await retryInterval(
+      6,
+      500,
+      async () => {
+        await this.mobileTap(centerX, centerY);
+        let webClickCoordinates;
+        let title;
+        try {
+          title = await this.title();
+          this.log.debug(JSON.stringify(title));
+          webClickCoordinates = _.isPlainObject(title) ? title : JSON.parse(title);
+        } catch (e) {
+          throw new Error(`${errorPrefix} Original error: ${e.message}`);
+        }
+        const {x, y} = webClickCoordinates;
+        if (!_.isInteger(x) || !_.isInteger(y)) {
+          throw new Error(errorPrefix);
+        }
+        let pixelRatio = 1;
+        if ((x > centerX) || (y > centerY)) {
+          // Webview coordinates might be scaled, lets apply a proper ratio there
+          pixelRatio = /** @type {number} */ (await this.execute('return window.devicePixelRatio;'));
+        }
+        this.webviewCalibrationResult = {
+          offsetX: centerX - x / pixelRatio,
+          offsetY: centerY - y / pixelRatio,
+          pixelRatio,
+        };
+      }
+    );
+
+    if (currentUrl) {
+      // restore the previous url
+      await this.setUrl(currentUrl);
+    }
+    // @ts-ignore it is always defined here
+    return this.webviewCalibrationResult;
+  },
+
   /**
    * @typedef {Object} SafariOpts
    * @property {object} preferences An object containing Safari settings to be updated.

lib/desired-caps.js

@@ -103,6 +103,9 @@ const desiredCapConstraints = /** @type {const} */ ({
   wdaLocalPort: {
     isNumber: true,
   },
+  wdaRemotePort: {
+    isNumber: true,
+  },
   wdaBaseUrl: {
     isString: true,
   },