workflow(ci): fix ai e2e test

packages/midscene/src/ai-model/openai/index.ts

@@ -43,7 +43,6 @@ if (typeof process.env[MIDSCENE_MODEL_NAME] === 'string') {
 async function createOpenAI() {
   let openai: OpenAI | AzureOpenAI;
   if (process.env[OPENAI_USE_AZURE]) {
-    console.log('Using Azure OpenAI');
     openai = new AzureOpenAI(extraConfig);
   } else {
     openai = new OpenAI(extraConfig);

packages/web-integration/playwright.config.ts

@@ -18,7 +18,7 @@ dotenv.config({
  */
 export default defineConfig({
   // testDir: './tests/ai/e2e',
-  // testIgnore: 'generate-test-data.spec.ts',
+  testIgnore: 'generate-test-data.spec.ts',
   timeout: 900 * 1000,
   /* Run tests in files in parallel */
   fullyParallel: false,

packages/web-integration/src/extractor/web-extractor.ts

@@ -60,6 +60,15 @@ function collectElementInfo(
       // Retrieve the text content of the selected option
       valueContent = selectedOption.textContent || '';
     }
+
+    if (
+      ((node as HTMLElement).tagName.toLowerCase() === 'input' ||
+        (node as HTMLElement).tagName.toLowerCase() === 'textarea') &&
+      (node as HTMLInputElement).value
+    ) {
+      valueContent = (node as HTMLInputElement).value;
+    }
+
     const elementInfo: WebElementInfo = {
       id: nodeHashId,
       nodePath,

packages/web-integration/src/puppeteer/base-page.ts

@@ -63,7 +63,6 @@ export class Page<
     });
 
     let buf: Buffer;
-    console.log(viewportSize);
     if (viewportSize.deviceScaleFactor > 1) {
       buf = (await resizeImg(readFileSync(path), {
         width: viewportSize.width,

packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt

@@ -1,14 +1,14 @@
 [
     {
-        "title": "Enter \"Learn\" in the task box",
+        "title": "Enter \"Learn\" in the task box, don't press enter",
         "actions": [
             "Planning",
             "Insight / Locate",
             "Action / Input"
         ]
     },
     {
-        "title": "Add \"JS today\" to the existing content of the task box and press enter",
+        "title": "Add \"JS today\" to base on the existing content(important) of the task box, then press enter",
         "actions": [
             "Planning",
             "Insight / Locate",
@@ -28,10 +28,7 @@
     {
         "title": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create",
         "actions": [
-            "Planning",
-            "Insight / Locate",
-            "Action / Input",
-            "Action / KeyboardPress"
+            "Planning"
         ]
     }
 ]

packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts

@@ -12,9 +12,10 @@ test('ai todo', async ({ ai, aiQuery }) => {
     test.setTimeout(1000 * 30);
   }
 
-  await ai('Enter "Learn" in the task box');
+  await ai('Enter "Learn" in the task box, don\'t press enter');
+
   await ai(
-    'Add "JS today" to the existing content of the task box and press enter',
+    'Add "JS today" to base on the existing content(important) of the task box, then press enter',
   );
 
   await ai(
@@ -33,7 +34,7 @@ test('ai todo', async ({ ai, aiQuery }) => {
   await ai('Move your mouse over the second item in the task list');
   await ai('Click the delete button to the right of the second task');
   await ai('Click the check button to the left of the second task');
-  await ai('Click the completed Status button below the task list');
+  await ai('Click the "completed" Status button below the task list');
 
   const taskList = await aiQuery<string[]>('string[], tasks in the list');
   expect(taskList.length).toBe(1);