fix(web-extract): fix the extractor for form item like <input />

apps/site/docs/zh/docs/usage/cli.md

@@ -13,6 +13,7 @@
 ```bash
 # 请替换为你自己的 API 密钥
 export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
+```
 
 相关文档：
 * [自定义模型服务](./model-vendor.html)

packages/midscene/src/action/executor.ts

@@ -147,13 +147,14 @@ export class Executor {
 
     if (successfullyCompleted) {
       this.status = 'completed';
-      if (this.tasks.length) {
-        // return the last output
-        return this.tasks[this.tasks.length - 1].output;
-      }
     } else {
       this.status = 'error';
     }
+    if (this.tasks.length) {
+      // return the last output
+      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
+      return this.tasks[outputIndex].output;
+    }
   }
 
   isInErrorState(): boolean {

packages/midscene/src/ai-model/openai/index.ts

@@ -8,6 +8,7 @@ export const MIDSCENE_OPENAI_INIT_CONFIG_JSON =
   'MIDSCENE_OPENAI_INIT_CONFIG_JSON';
 export const MIDSCENE_MODEL_NAME = 'MIDSCENE_MODEL_NAME';
 export const MIDSCENE_LANGSMITH_DEBUG = 'MIDSCENE_LANGSMITH_DEBUG';
+export const MIDSCENE_DEBUG_AI_PROFILE = 'MIDSCENE_DEBUG_AI_PROFILE';
 export const OPENAI_API_KEY = 'OPENAI_API_KEY';
 
 export function useOpenAIModel(useModel?: 'coze' | 'openAI') {
@@ -26,6 +27,7 @@ if (
   extraConfig = JSON.parse(process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON]);
 }
 
+// default model
 let model = 'gpt-4o';
 if (typeof process.env[MIDSCENE_MODEL_NAME] === 'string') {
   console.log(`model: ${process.env[MIDSCENE_MODEL_NAME]}`);
@@ -49,12 +51,18 @@ export async function call(
   responseFormat?: AIResponseFormat,
 ): Promise<string> {
   const openai = await createOpenAI();
+
+  const shouldPrintTiming =
+    typeof process.env[MIDSCENE_DEBUG_AI_PROFILE] === 'string';
+  shouldPrintTiming && console.time('Midscene - AI call');
   const completion = await openai.chat.completions.create({
     model,
     messages,
     response_format: { type: responseFormat },
     temperature: 0.2,
   });
+  shouldPrintTiming && console.timeEnd('Midscene - AI call');
+  shouldPrintTiming && console.log('Midscene - AI usage', completion.usage);
 
   const { content } = completion.choices[0].message;
   assert(content, 'empty content');

packages/midscene/src/image/index.ts

@@ -1,7 +1,6 @@
 export { imageInfo, imageInfoOfBase64, base64Encoded } from './info';
 export {
   alignCoordByTrim,
-  trimImage,
   calculateNewDimensions,
   resizeImg,
   transformImgPathToBase64,

packages/midscene/src/image/transform.ts

@@ -119,49 +119,6 @@ export function calculateNewDimensions(
   };
 }
 
-/**
- * Trims an image and returns the trimming information, including the offset from the left and top edges, and the trimmed width and height
- *
- * @param image - The image to be trimmed. This can be a file path or a Buffer object containing the image data
- * @returns A Promise that resolves to an object containing the trimming information. If the image does not need to be trimmed, this object will be null
- */
-export async function trimImage(image: string | Buffer): Promise<{
-  trimOffsetLeft: number; // attention: trimOffsetLeft is a negative number
-  trimOffsetTop: number; // so as trimOffsetTop
-  width: number;
-  height: number;
-} | null> {
-  const imgInstance = Sharp(image);
-  const instanceInfo = await imgInstance.metadata();
-
-  if (
-    !instanceInfo.width ||
-    instanceInfo.width <= 3 ||
-    !instanceInfo.height ||
-    instanceInfo.height <= 3
-  ) {
-    return null;
-  }
-
-  const { info } = await imgInstance.trim().toBuffer({
-    resolveWithObject: true,
-  });
-
-  if (
-    typeof info.trimOffsetLeft === 'undefined' ||
-    typeof info.trimOffsetTop === 'undefined'
-  ) {
-    return null;
-  }
-
-  return {
-    trimOffsetLeft: info.trimOffsetLeft,
-    trimOffsetTop: info.trimOffsetTop,
-    width: info.width,
-    height: info.height,
-  };
-}
-
 /**
  * Aligns an image's coordinate system based on trimming information
  *
@@ -179,10 +136,16 @@ export async function trimImage(image: string | Buffer): Promise<{
  * @throws Error if there is an error during image processing
  */
 export async function alignCoordByTrim(
-  image: string | Buffer,
+  image: string | Buffer | Sharp.Sharp,
   centerRect: Rect,
 ): Promise<Rect> {
-  const imgInfo = await Sharp(image).metadata();
+  // const img = await Sharp(image); // .webp();
+  const img: Sharp.Sharp =
+    typeof image === 'string' || Buffer.isBuffer(image)
+      ? Sharp(image)
+      : image.clone();
+  const imgInfo = await img.metadata();
+
   if (
     !imgInfo?.width ||
     !imgInfo.height ||
@@ -191,21 +154,64 @@ export async function alignCoordByTrim(
   ) {
     return centerRect;
   }
+
+  const zeroSize: Rect = {
+    left: 0,
+    top: 0,
+    width: -1,
+    height: -1,
+  };
+  const finalCenterRect: Rect = { ...centerRect };
+  if (centerRect.left > imgInfo.width || centerRect.top > imgInfo.height) {
+    return zeroSize;
+  }
+
+  if (finalCenterRect.left < 0) {
+    finalCenterRect.width += finalCenterRect.left;
+    finalCenterRect.left = 0;
+  }
+
+  if (finalCenterRect.top < 0) {
+    finalCenterRect.height += finalCenterRect.top;
+    finalCenterRect.top = 0;
+  }
+
+  if (finalCenterRect.left + finalCenterRect.width > imgInfo.width) {
+    finalCenterRect.width = imgInfo.width - finalCenterRect.left;
+  }
+  if (finalCenterRect.top + finalCenterRect.height > imgInfo.height) {
+    finalCenterRect.height = imgInfo.height - finalCenterRect.top;
+  }
+
+  if (finalCenterRect.width <= 3 || finalCenterRect.height <= 3) {
+    return finalCenterRect;
+  }
+
   try {
-    const img = await Sharp(image).extract(centerRect).toBuffer();
-    const trimInfo = await trimImage(img);
-    if (!trimInfo) {
-      return centerRect;
+    const croppedImg = await img
+      .extract(finalCenterRect)
+      .jpeg({
+        quality: 75,
+      })
+      .toBuffer();
+    const { info: trimInfo } = await Sharp(croppedImg).trim().toBuffer({
+      resolveWithObject: true,
+    });
+    if (
+      !trimInfo ||
+      typeof trimInfo.trimOffsetLeft === 'undefined' ||
+      typeof trimInfo.trimOffsetTop === 'undefined'
+    ) {
+      return finalCenterRect;
     }
-
     return {
-      left: centerRect.left - trimInfo.trimOffsetLeft,
-      top: centerRect.top - trimInfo.trimOffsetTop,
+      left: finalCenterRect.left - trimInfo.trimOffsetLeft,
+      top: finalCenterRect.top - trimInfo.trimOffsetTop,
       width: trimInfo.width,
       height: trimInfo.height,
     };
   } catch (e) {
-    console.log(imgInfo);
+    console.warn(imgInfo, finalCenterRect);
     throw e;
   }
 }

packages/midscene/src/types.ts

@@ -15,7 +15,7 @@ export interface Size {
 export type Rect = Point & Size;
 
 enum NodeType {
-  INPUT = 'INPUT Node',
+  FORM_ITEM = 'FORM_ITEM Node',
   BUTTON = 'BUTTON Node',
   IMG = 'IMG Node',
   TEXT = 'TEXT Node',

packages/midscene/tests/ai/executor/index.test.ts

@@ -23,8 +23,10 @@ const insightFindTask = (shouldThrow?: boolean) => {
     param: {
       prompt: 'test',
     },
-    async executor(param) {
+    async executor(param, taskContext) {
       if (shouldThrow) {
+        const { task } = taskContext;
+        task.output = 'error-output';
         await new Promise((resolve) => setTimeout(resolve, 100));
         throw new Error('test-error');
       }
@@ -179,66 +181,8 @@ describe('executor', () => {
     expect(tasks[1].status).toBe('cancelled');
     expect(executor.status).toBe('error');
     expect(executor.latestErrorTask()).toBeTruthy();
-    expect(r).toBeFalsy();
-
-    // expect to throw an error
-    expect(async () => {
-      await executor.flush();
-    }).rejects.toThrowError();
-
-    expect(async () => {
-      await executor.append(insightFindTask());
-    }).rejects.toThrowError();
-  });
-
-  it.skip('insight - return error instead of throwing', async () => {
-    const executor = new Executor('test', 'test-description', [
-      {
-        type: 'Insight',
-        subType: 'Locate',
-        param: {
-          prompt: 'test',
-        },
-        async executor(param) {
-          return {
-            output: {
-              element: 'abc',
-            },
-            log: {
-              dump: {},
-            },
-          };
-        },
-      },
-      {
-        type: 'Insight',
-        subType: 'Locate',
-        param: {
-          prompt: 'test',
-        },
-        async executor(param) {
-          return {
-            output: {
-              element: 'abc',
-            },
-            log: {
-              dump: {},
-            },
-          };
-        },
-      },
-    ]);
-    const r = await executor.flush();
-    const tasks = executor.tasks as ExecutionTaskInsightLocate[];
-
-    expect(tasks.length).toBe(2);
-    expect(tasks[0].status).toBe('failed');
-    expect(tasks[0].error).toBeTruthy();
-    expect(tasks[0].timing!.end).toBeTruthy();
-    expect(tasks[1].status).toBe('cancelled');
-    expect(executor.status).toBe('error');
-    expect(executor.latestErrorTask()).toBeTruthy();
-    expect(r).toBeFalsy();
+    expect(executor.isInErrorState()).toBeTruthy();
+    expect(r).toEqual('error-output');
 
     // expect to throw an error
     expect(async () => {

packages/midscene/tests/unit-test/image/__snapshots__/index.test.ts.snap

@@ -9,6 +9,42 @@ exports[`image utils > align a sub-image 1`] = `
 }
 `;
 
+exports[`image utils > align a sub-image with negative coord 1`] = `
+{
+  "height": 100,
+  "left": 0,
+  "top": 0,
+  "width": 100,
+}
+`;
+
+exports[`image utils > align a table style sub-image 1`] = `
+{
+  "height": 57,
+  "left": 140,
+  "top": 73,
+  "width": 200,
+}
+`;
+
+exports[`image utils > align a tiny sub-image 1`] = `
+{
+  "height": 80,
+  "left": 140,
+  "top": 50,
+  "width": 200,
+}
+`;
+
+exports[`image utils > align an oversized sub-image 1`] = `
+{
+  "height": 50,
+  "left": 2860,
+  "top": 200,
+  "width": 2,
+}
+`;
+
 exports[`image utils > base64 + imageInfo 1`] = `
 {
   "height": 56,
@@ -20,6 +56,15 @@ exports[`image utils > base64Encoded 1`] = `"data:image/png;base64,iVBORw0KGgoAA
 
 exports[`image utils > base64Encoded 2`] = `"iVBORw0KGgoAAAANSUhEUgAAAEQAAAA4CAYAAABE814IAAAKqGlDQ1BJQ0MgUHJvZmlsZQAASImVlwdUk9kSgO//p4eEltBb6E2QTgApoYcivdoISYBQQggEBTuyqMBaUBHBsiKLAgouKiBrQRSxsAhYwLogi4C6bizYUHk/cAjuvvPeO29y5twv88+dmfufOzkTAMhUlkCQCssCkMbPEob6uNOiY2JpuDGABgRABErAlMXOFDCCgwMAInPr3+X9PQBNr7fNpmP9+/P/KnIcbiYbACgY4XhOJjsN4dOIitkCYRYAqCrErrsySzDN1xCmCpECEX40zYmzLJ7m+BlGo2d8wkM9EFYGAE9isYSJAJD0EDstm52IxCF5ImzB5/D4CCPfgUtaWjoHYSQvMEJ8BAhPx6fHfxcn8W8x4yUxWaxECc+eZUbwnrxMQSor5/98Hf9b0lJFczkMECUlCX1DkRWpCxpISfeXMD9+cdAc8zgz/jOcJPKNmGN2pkfsHHNYnv6SvamLA+Y4gefNlMTJYobPMTfTK2yOhemhklwJQg/GHLOE83lFKRESexKXKYmfmxQeNcfZvMjFc5yZEuY/7+MhsQtFoZL6uXwf9/m83pKzp2V+d14eU7I3KyncV3J21nz9XD5jPmZmtKQ2DtfTa94nQuIvyHKX5BKkBkv8uak+EntmdphkbxZyIef3BkveYTLLL3iOgSfwAgHIhwYigBWwR9QahACvLO6q6TsKPNIFOUJeYlIWjYF0GZfG5LPNF9CsLKxsAJju2dkr8XZgphchRfy8LT8PgEVTCNyatwUi2rgLuT6r5236SF2ySE90/MEWCbNnbdPtBDDIL4EMoAIVoAl0gREwQ2qzA07ADanYDwSBcBADlgM2SAJpQAhWgjVgIygARWAH2APKwSFwBBwDJ0AjaAbnwCVwFdwEPeAueAgGwQh4AcTgPZiEIAgHkSEKpAJpQfqQKWQF0SEXyAsKgEKhGCgOSoT4kAhaA22CiqASqBw6DNVAv0BnoUvQdagXug8NQePQG+gzjIJJMBXWgA3ghTAdZsD+cDi8DE6EM+BcOB/eBpfBlfBxuAm+BN+E78KD8At4AgVQUihFlDbKDEVHeaCCULGoBJQQtQ5ViCpFVaLqUa2oTtRt1CDqJeoTGoumoGloM7QT2hcdgWajM9Dr0MXocvQxdBP6Cvo2eggtRn/DkDHqGFOMI4aJicYkYlZiCjClmGrMGUwH5i5mBPMei8UqYg2x9lhfbAw2GbsaW4w9gG3AtmF7scPYCRwOp4IzxTnjgnAsXBauALcPdxx3EdeHG8F9xEvhtfBWeG98LJ6Pz8OX4mvxF/B9+FH8JEGWoE9wJAQROIQcwnZCFaGVcIswQpgkyhENic7EcGIycSOxjFhP7CA+Ir6VkpLSkXKQCpHiSW2QKpM6KXVNakjqE0meZELyIC0liUjbSEdJbaT7pLdkMtmA7EaOJWeRt5FryJfJT8gfpSnS5tJMaY70eukK6SbpPulXMgQZfRmGzHKZXJlSmVMyt2ReyhJkDWQ9ZFmy62QrZM/K9stOyFHkLOWC5NLkiuVq5a7Ljcnj5A3kveQ58vnyR+Qvyw9TUBRdigeFTdlEqaJ0UEaoWKohlUlNphZRT1C7qWIFeQUbhUiFVQoVCucVBhVRigaKTMVUxe2KjYr3FD8raSgxlLhKW5XqlfqUPiirKbspc5ULlRuU7yp/VqGpeKmkqOxUaVZ5rIpWNVENUV2pelC1Q/WlGlXNSY2tVqjWqPZAHVY3UQ9VX61+RL1LfUJDU8NHQ6CxT+OyxktNRU03zWTN3ZoXNMe1KFouWjyt3VoXtZ7TFGgMWiqtjHaFJtZW1/bVFmkf1u7WntQx1InQydNp0HmsS9Sl6ybo7tZt1xXraekF6q3Rq9N7oE/Qp+sn6e/V79T/YGBoEGWw2aDZYMxQ2ZBpmGtYZ/jIiGzkapRhVGl0xxhrTDdOMT5g3GMCm9iaJJlUmNwyhU3tTHmmB0x7F2AWOCzgL6hc0G9GMmOYZZvVmQ2ZK5oHmOeZN5u/Wqi3MHbhzoWdC79Z2FqkWlRZPLSUt/SzzLNstXxjZWLFtqqwumNNtva2Xm/dYv3axtSGa3PQZsCWYhtou9m23farnb2d0K7ebtxezz7Ofr99P51KD6YX0685YBzcHdY7nHP45GjnmOXY6PiXk5lTilOt09giw0XcRVWLhp11nFnOh50HXWgucS4/uQy6aruyXCtdn7rpunHcqt1GGcaMZMZxxit3C3eh+xn3Dx6OHms92jxRnj6ehZ7dXvJeEV7lXk+8dbwTveu8xT62Pqt92nwxvv6+O337mRpMNrOGKfaz91vrd8Wf5B/mX+7/NMAkQBjQGggH+gXuCny0WH8xf3FzEAhiBu0KehxsGJwR/GsINiQ4pCLkWahl6JrQzjBK2Iqw2rD34e7h28MfRhhFiCLaI2Uil0bWRH6I8owqiRqMXhi9NvpmjGoML6YlFhcbGVsdO7HEa8meJSNLbZcWLL23zHDZqmXXl6suT11+foXMCtaKU3GYuKi42rgvrCBWJWsinhm/P17M9mDvZb/guHF2c8a5ztwS7miCc0JJwliic+KuxPEk16TSpJc8D14573Wyb/Kh5A8pQSlHU6ZSo1Ib0vBpcWln+fL8FP6VdM30Vem9AlNBgWAwwzFjT4ZY6C+szoQyl2W2ZFGR4ahLZCT6QTSU7ZJdkf1xZeTKU6vkVvFXdeWY5GzNGc31zv15NXo1e3X7Gu01G9cMrWWsPbwOWhe/rn297vr89SMbfDYc20jcmLLxtzyLvJK8d5uiNrXma+RvyB/+weeHugLpAmFB/2anzYe2oLfwtnRvtd66b+u3Qk7hjSKLotKiL8Xs4hs/Wv5Y9uPUtoRt3dvtth/cgd3B33Fvp+vOYyVyJbklw7sCdzXtpu0u3P1uz4o910ttSg/tJe4V7R0sCyhr2ae3b8e+L+VJ5Xcr3Csa9qvv37r/wwHOgb6DbgfrD2kcKjr0+SfeTwOHfQ43VRpUlh7BHsk+8qwqsqrzZ/rPNdWq1UXVX4/yjw4eCz12pca+pqZWvXZ7HVwnqhs/vvR4zwnPEy31ZvWHGxQbik6Ck6KTz3+J++Veo39j+yn6qfrT+qf3n6GcKWyCmnKaxM1JzYMtMS29Z/3Otrc6tZ751fzXo+e0z1WcVzi//QLxQv6FqYu5FyfaBG0vLyVeGm5f0f7wcvTlO1dCrnR3+Hdcu+p99XIno/PiNedr5647Xj97g36j+abdzaYu264zv9n+dqbbrrvplv2tlh6HntbeRb0X+lz7Lt32vH31DvPOzbuL7/bei7g30L+0f3CAMzB2P/X+6wfZDyYfbniEeVT4WPZx6RP1J5W/G//eMGg3eH7Ic6jradjTh8Ps4Rd/ZP7xZST/GflZ6ajWaM2Y1di5ce/xnudLno+8ELyYfFnwp9yf+18ZvTr9l9tfXeJo8chr4eupN8VvVd4efWfzrn0ieOLJ+7T3kx8KP6p8PPaJ/qnzc9Tn0cmVX3Bfyr4af2395v/t0VTa1JSAJWTNjAIoROGEBADeHAWAHAMApQcA4pLZmXpGoNn/ATME/hPPzt0zYgfAiTYAgt0A8GybZX3ELOM2awt3A7C1tUTn5t+ZWX1aZI8D4J5jR7cPGNriBP4ps3P8d3X/cwWSqH9b/wVg3QX8UeufjwAAAIplWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACQAAAAAQAAAJAAAAABAAOShgAHAAAAEgAAAHigAgAEAAAAAQAAAESgAwAEAAAAAQAAADgAAAAAQVNDSUkAAABTY3JlZW5zaG90skJVjwAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAdRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+NTY8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+Njg8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpVc2VyQ29tbWVudD5TY3JlZW5zaG90PC9leGlmOlVzZXJDb21tZW50PgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KoAu3GgAAABxpRE9UAAAAAgAAAAAAAAAcAAAAKAAAABwAAAAcAAAA+gvIlt0AAADGSURBVGgF7NIrDsIAFETR1xRBICCQhI9mAVi2gEKxFywLQeGxWLaBJUEhGmj6gQQxCZMaXMWtejc17ckks/nkHTwSSACRxfcA5NcjAAHEBCxZCCAmYMlCADEBSxYCiAlYshBATMCShQBiApYsBBATsGQhgJiAJQtpO0h/NNAn1lUdz0embjrSThrdYU+vyryIPHup/z1at5D1bqt/qIoyTvujuukYL6ax3Kz06n69xeVwVv97AGJigABiApYsBBATsGQhBvIBAAD//8WZ0QUAAAHhSURBVO2UzStEURjGn+tjEml8jQaDaeQrrIYIzWIkSVGKBcrCyn9gw07KxtbeAikpUpSw0VigpIgmJWqGhc9hPgxm1D11X9RczeLQezfnfd7zdu95fj33KEXFlndI9HSN9YvThEOvWJmYF/q7Ir+yEHU9DrF1c+7BzsyG0HoLhYFokTEQLQ8wEAZCCBApdULewmEsj8+RI2ulpdoKe3eTaP67S7VztA+KogiDa1OL8D++CE2LmvZa2OrKRdtzeonduW2h9RbSJaRjpBdJhmTh42DZhYsDt9C0cA53IN2UIdpu1zGO1veF1ltIB6R5sBXZxbnCx2swhM3pVTzfPYmeWtjqK1DTZlfl5+qa3YL37ErT0yOkA2KymdE40KLxEIVyvHmI64jR53sfsgpzUFBlhdVeqpkL+YNYnVzQ9PQK6YBEDTQOOGGy5en1Erk7tuA5/X06oh+UEkhCYiIcQ20wmjNjhnK0vge36yTm+Z8GpQSiHrakoRJljmoYUgxq68v64L3F3tIOHrx3X/Z+05AaiGrIaM76vDdSjWmIpifoD0QuWR+8kd8j+BJQx+Ky/gkgcXEa40sYCAHFQBgIIUAkJ4SBEAJEckIYCCFAJCeEgRACRHJCGAghQCQnhIEQAkR+AOdaTYjLuDCrAAAAAElFTkSuQmCC"`;
 
+exports[`image utils > illegal center rect, refuse to align 1`] = `
+{
+  "height": -1,
+  "left": 0,
+  "top": 0,
+  "width": -1,
+}
+`;
+
 exports[`image utils > imageInfo 1`] = `
 {
   "height": 56,