test(playwright): refactor to use artillery website to be more reliab…

…le (#2502)

* test(playwright): increase expect timeout on test

* test(playwright): refactor tests to use artillery website

* refactor: add a visibility check

* refactor: add retry to url navigation due to CI bug

* test(playwright): remove unnecessary comments

* test(playwright): add logs in retries to help surface the issue

packages/artillery-engine-playwright/test/fargate.aws.js

@@ -3,7 +3,7 @@ const { $ } = require('zx');
 const { getTestTags } = require('../../artillery/test/cli/_helpers.js');
 const fs = require('fs');
 
-const TEST_URL = 'https://github.com/artilleryio/artillery/';
+const TEST_URL = 'https://www.artillery.io/';
 const tags = getTestTags(['typescript:true']);
 let playwrightOutput;
 
@@ -59,12 +59,21 @@ test('playwright typescript test works and reports data', async (t) => {
   //Assert: reports steps as histograms
   t.hasProp(
     summaries,
-    'browser.step.go_to_artillery_repo',
-    'should have reported step go_to_artillery_repo as histogram'
+    'browser.step.go_to_artillery_io',
+    'should have reported step go_to_artillery_io as histogram'
   );
   t.ok(
-    Object.keys(summaries['browser.step.go_to_artillery_repo']).includes('p99'),
-    'should have reported step go_to_artillery_repo as histogram with p99 metric'
+    Object.keys(summaries['browser.step.go_to_artillery_io']).includes('p99'),
+    'should have reported step go_to_artillery_io as histogram with p99 metric'
+  );
+  t.hasProp(
+    summaries,
+    'browser.step.go_to_docs',
+    'should have reported step go_to_docs as histogram'
+  );
+  t.ok(
+    Object.keys(summaries['browser.step.go_to_docs']).includes('p99'),
+    'should have reported step go_to_docs as histogram with p99 metric'
   );
 
   //Assert: reports web vital metrics

packages/artillery-engine-playwright/test/fixtures/processor.js

@@ -1,16 +1,47 @@
 const { expect } = require('@playwright/test');
 
+//this is due to occasional failures in CI due to known unresolved issue: https://github.com/microsoft/playwright/issues/13062
+const retryGoingToPage = async (page, url) => {
+  let retryCount = 0;
+  let error;
+  while (retryCount < 5) {
+    try {
+      await page.goto(url);
+      return;
+    } catch (err) {
+      console.log(`ERROR: page.goto in Playwright test - ${err.message}`);
+      console.log('Retrying...');
+      error = err;
+      retryCount++;
+    }
+  }
+  throw new Error(`Failed to go to page ${url}: ${error}`);
+};
+
 async function artilleryPlaywrightFunction(page, vuContext, events, test) {
-  await test.step('go_to_artillery_repo', async () => {
-    await page.goto(`${vuContext.vars.target}/`);
-    await expect(page.getByTestId('latest-commit')).toBeVisible();
+  await test.step('go_to_artillery_io', async () => {
+    await retryGoingToPage(page, '/');
+
+    await expect(page.getByText('The Artillery Manifesto')).toBeVisible();
   });
+
+  await test.step('go_to_docs', async () => {
+    await page
+      .getByLabel('Main navigation')
+      .getByRole('link', { name: 'Documentation' })
+      .click();
+    await expect(page).toHaveURL('/docs');
+    await expect(
+      page.getByText("What's different about Artillery?")
+    ).toBeVisible();
+  });
+
   events.emit('counter', 'custom_emitter', 1);
 }
 
 async function playwrightFunctionWithFailure(page, vuContext, events, test) {
-  await test.step('go_to_artillery_repo', async () => {
-    await page.goto(`${vuContext.vars.target}/`);
+  await test.step('go_to_artillery_io', async () => {
+    await retryGoingToPage(page, '/');
     await expect(page.getByText('gremlins are here!')).toBeVisible();
   });
   events.emit('counter', 'custom_emitter', 1);

packages/artillery-engine-playwright/test/fixtures/processor.ts

@@ -1,14 +1,43 @@
 import { expect, Page } from '@playwright/test';
 
+//this is due to occasional failures in CI due to known unresolved issue (ERR_NETWORK_CHANGED): https://github.com/microsoft/playwright/issues/13062
+const retryGoingToPage = async (page, url) => {
+  let retryCount = 0;
+  let error;
+  while (retryCount < 10) {
+    try {
+      await page.goto(url);
+      return;
+    } catch (err) {
+      console.log(`ERROR: page.goto in Playwright test - ${err.message}`);
+      console.log(`Retrying...`);
+      error = err;
+      retryCount++;
+    }
+  }
+  throw new Error(`Failed to go to page ${url}: ${error}`);
+};
+
 export async function artilleryPlaywrightFunction(
   page: Page,
   vuContext,
   events,
   test
 ) {
-  await test.step('go_to_artillery_repo', async () => {
-    await page.goto(`${vuContext.vars.target}/`);
-    await expect(page.getByTestId('latest-commit')).toBeVisible();
+  await test.step('go_to_artillery_io', async () => {
+    await retryGoingToPage(page, '/');
+    await expect(page.getByText('The Artillery Manifesto')).toBeVisible();
+  });
+
+  await test.step('go_to_docs', async () => {
+    await page
+      .getByLabel('Main navigation')
+      .getByRole('link', { name: 'Documentation' })
+      .click();
+    await expect(page).toHaveURL('/docs');
+    await expect(
+      page.getByText("What's different about Artillery?")
+    ).toBeVisible();
   });
 }
 
@@ -18,8 +47,8 @@ export async function playwrightFunctionWithFailure(
   events,
   test
 ) {
-  await test.step('go_to_artillery_repo', async () => {
-    await page.goto(`${vuContext.vars.target}/`);
+  await test.step('go_to_artillery_io', async () => {
+    await retryGoingToPage(page, '/');
     await expect(page.getByText('gremlins are here!')).toBeVisible();
   });
   events.emit('counter', 'custom_emitter', 1);

packages/artillery-engine-playwright/test/fixtures/pw-acceptance.yml

@@ -1,5 +1,5 @@
 config:
-  target: "https://github.com/artilleryio/artillery"
+  target: "https://www.artillery.io/"
   phases:
     - duration: 3
       arrivalRate: 1

packages/artillery-engine-playwright/test/index.test.js

@@ -2,7 +2,7 @@ const { test, afterEach, beforeEach } = require('tap');
 const { $ } = require('zx');
 const fs = require('fs');
 
-const TEST_URL = 'https://github.com/artilleryio/artillery/';
+const TEST_URL = 'https://www.artillery.io/';
 let playwrightOutput;
 
 beforeEach(() => {
@@ -54,12 +54,21 @@ test('playwright js test works and reports data', async (t) => {
   //Assert: reports steps as histograms
   t.hasProp(
     summaries,
-    'browser.step.go_to_artillery_repo',
-    'should have reported step go_to_artillery_repo as histogram'
+    'browser.step.go_to_artillery_io',
+    'should have reported step go_to_artillery_io as histogram'
   );
   t.ok(
-    Object.keys(summaries['browser.step.go_to_artillery_repo']).includes('p99'),
-    'should have reported step go_to_artillery_repo as histogram with p99 metric'
+    Object.keys(summaries['browser.step.go_to_artillery_io']).includes('p99'),
+    'should have reported step go_to_artillery_io as histogram with p99 metric'
+  );
+  t.hasProp(
+    summaries,
+    'browser.step.go_to_docs',
+    'should have reported step go_to_docs as histogram'
+  );
+  t.ok(
+    Object.keys(summaries['browser.step.go_to_docs']).includes('p99'),
+    'should have reported step go_to_docs as histogram with p99 metric'
   );
 
   //Assert: reports web vital metrics
@@ -180,12 +189,22 @@ test('playwright typescript test works and reports data', async (t) => {
   //Assert: reports steps as histograms
   t.hasProp(
     summaries,
-    'browser.step.go_to_artillery_repo',
-    'should have reported step go_to_artillery_repo as histogram'
+    'browser.step.go_to_artillery_io',
+    'should have reported step go_to_artillery_io as histogram'
+  );
+  t.ok(
+    Object.keys(summaries['browser.step.go_to_artillery_io']).includes('p99'),
+    'should have reported step go_to_artillery_io as histogram with p99 metric'
+  );
+
+  t.hasProp(
+    summaries,
+    'browser.step.go_to_docs',
+    'should have reported step go_to_artillery_io as histogram'
   );
   t.ok(
-    Object.keys(summaries['browser.step.go_to_artillery_repo']).includes('p99'),
-    'should have reported step go_to_artillery_repo as histogram with p99 metric'
+    Object.keys(summaries['browser.step.go_to_docs']).includes('p99'),
+    'should have reported step go_to_docs as histogram with p99 metric'
   );
 
   //Assert: reports web vital metrics