feat: supports control of iOS and Android devices through appium

packages/midscene/src/ai-model/automation/planning.ts

@@ -17,7 +17,7 @@ export function systemPromptToTaskPlanning() {
   * type: 'KeyboardPress',  press a key
     * param: { value: string },  the value to input or the key to press. Use （Enter, Shift, Control, Alt, Meta, ShiftLeft, ControlOrMeta, ControlOrMeta） to represent the key.
   * type: 'Scroll'
-    * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' }
+    * param: { scrollType: 'scrollDownOneScreen', 'scrollUpOneScreen', 'scrollUntilBottom', 'scrollUntilTop' }
   * type: 'Error'
     * param: { message: string }, the error message
   * type: 'Sleep'

packages/midscene/src/types.ts

@@ -209,10 +209,10 @@ export interface PlanningActionParamInputOrKeyPress {
 }
 export interface PlanningActionParamScroll {
   scrollType:
-    | 'ScrollUntilBottom'
-    | 'ScrollUntilTop'
-    | 'ScrollDown'
-    | 'ScrollUp';
+    | 'scrollUntilTop'
+    | 'scrollUntilBottom'
+    | 'scrollUpOneScreen'
+    | 'scrollDownOneScreen';
 }
 
 export interface PlanningActionParamAssert {

packages/midscene/src/utils.ts

@@ -1,12 +1,6 @@
 import assert from 'node:assert';
 import { randomUUID } from 'node:crypto';
-import {
-  copyFileSync,
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  writeFileSync,
-} from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { basename, dirname, join } from 'node:path';
 import type { Rect, ReportDumpWithAttributes } from './types';

packages/shared/src/img/transform.ts

@@ -1,6 +1,5 @@
 import { Buffer } from 'node:buffer';
 import Jimp from 'jimp';
-import type { Rect } from '../types';
 
 /**
  * Saves a Base64-encoded image to a file
@@ -47,24 +46,27 @@ export async function transformImgPathToBase64(inputPath: string) {
  * @returns A Promise that resolves to a base64-encoded string representing the resized image
  * @throws An error if the width or height cannot be determined from the metadata
  */
-export async function resizeImg(base64Data: string) {
-  // Remove the base64 data prefix (if any)
-  const base64Image = base64Data.split(';base64,').pop() || base64Data;
-
-  // Converts base64 data to buffer
-  const imageBuffer = Buffer.from(base64Image, 'base64');
+export async function resizeImg(
+  inputData: string | Buffer,
+): Promise<string | Buffer> {
+  const isBase64 = typeof inputData === 'string';
+  const imageBuffer = isBase64
+    ? Buffer.from(inputData.split(';base64,').pop() || inputData, 'base64')
+    : inputData;
 
   const image = await Jimp.read(imageBuffer);
   const { width, height } = image.bitmap;
+
   if (!width || !height) {
-    throw Error('undefined width or height with url');
+    throw Error('Undefined width or height from the input image.');
   }
 
   const newSize = calculateNewDimensions(width, height);
 
   image.resize(newSize.width, newSize.height);
-  const buffer = await image.getBufferAsync(Jimp.MIME_PNG);
-  return buffer.toString('base64');
+  const resizedBuffer = await image.getBufferAsync(Jimp.MIME_PNG);
+
+  return isBase64 ? resizedBuffer.toString('base64') : resizedBuffer;
 }
 
 /**

packages/web-integration/package.json

@@ -77,32 +77,39 @@
     "openai": "4.47.1",
     "inquirer": "10.1.5",
     "@midscene/core": "workspace:*",
-    "@midscene/shared": "workspace:*"
+    "@midscene/shared": "workspace:*",
+    "@xmldom/xmldom": "0.8.10"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.58.2",
-    "js-sha256": "0.11.0",
+    "@playwright/test": "1.44.1",
+    "@types/fs-extra": "11.0.4",
     "@types/node": "^18.0.0",
-    "typescript": "~5.0.4",
-    "vitest": "^1.6.0",
+    "@wdio/types": "9.0.4",
+    "dotenv": "16.4.5",
+    "fs-extra": "11.2.0",
+    "js-sha256": "0.11.0",
     "playwright": "1.44.1",
     "puppeteer": "23.0.2",
-    "@playwright/test": "1.44.1",
-    "fs-extra": "11.2.0",
-    "@types/fs-extra": "11.0.4",
-    "dotenv": "16.4.5"
+    "typescript": "~5.0.4",
+    "vitest": "^1.6.0",
+    "webdriverio": "9.0.6"
   },
   "peerDependencies": {
     "@playwright/test": "^1.44.1",
     "playwright": "^1.44.1",
-    "puppeteer": ">=20.0.0"
+    "puppeteer": ">=20.0.0",
+    "webdriverio": ">=9.0.6"
   },
   "peerDependenciesMeta": {
     "@playwright/test": {
       "optional": true
     },
     "puppeteer": {
       "optional": true
+    },
+    "webdriverio": {
+      "optional": true
     }
   },
   "engines": {

packages/web-integration/src/Page.ts

@@ -0,0 +1,45 @@
+import type { WebKeyInput } from './common/page';
+import type { ElementInfo } from './extractor/extractor';
+
+type encodingType = 'base64' | 'binary';
+type imageType = 'jpeg' | 'png';
+
+export type screenshotOptions = {
+  path?: string;
+  encoding?: encodingType;
+  type?: imageType;
+  quality?: number;
+};
+export type MouseButton = 'left' | 'right' | 'middle';
+
+export abstract class AbstractPage {
+  abstract screenshot(options: screenshotOptions): Promise<Uint8Array>;
+  abstract getElementInfos(): Promise<ElementInfo[]>;
+  abstract url(): string;
+
+  get mouse() {
+    return {
+      click: async (
+        x: number,
+        y: number,
+        options: { button: MouseButton },
+      ) => {},
+      wheel: async (deltaX: number, deltaY: number) => {},
+      move: async (x: number, y: number) => {},
+    };
+  }
+
+  get keyboard() {
+    return {
+      type: async (text: string) => {},
+      press: async (key: WebKeyInput) => {},
+    };
+  }
+
+  async selectAll(): Promise<void> {}
+
+  abstract scrollUntilTop(): Promise<void>;
+  abstract scrollUntilBottom(): Promise<void>;
+  abstract scrollUpOneScreen(): Promise<void>;
+  abstract scrollDownOneScreen(): Promise<void>;
+}

packages/web-integration/src/appium/index.ts

@@ -0,0 +1,2 @@
+export { PageAgent as AppiumAgent } from '@/common/agent';
+export { Page as AppiumPage } from './page';