From f6a6aa82540fc4aca2577283cbbf598273e84c8f Mon Sep 17 00:00:00 2001 From: Bernardo Guerreiro <39738771+bernardobridge@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:07:58 +0000 Subject: [PATCH] test(playwright): refactor to use artillery website to be more reliable (#2502) * test(playwright): increase expect timeout on test * test(playwright): refactor tests to use artillery website * refactor: add a visibility check * refactor: add retry to url navigation due to CI bug * test(playwright): remove unnecessary comments * test(playwright): add logs in retries to help surface the issue --- .../test/fargate.aws.js | 19 ++++++--- .../test/fixtures/processor.js | 41 ++++++++++++++++--- .../test/fixtures/processor.ts | 39 +++++++++++++++--- .../test/fixtures/pw-acceptance.yml | 2 +- .../test/index.test.js | 37 +++++++++++++---- 5 files changed, 113 insertions(+), 25 deletions(-) diff --git a/packages/artillery-engine-playwright/test/fargate.aws.js b/packages/artillery-engine-playwright/test/fargate.aws.js index 6994e73c2e..e45b581252 100644 --- a/packages/artillery-engine-playwright/test/fargate.aws.js +++ b/packages/artillery-engine-playwright/test/fargate.aws.js @@ -3,7 +3,7 @@ const { $ } = require('zx'); const { getTestTags } = require('../../artillery/test/cli/_helpers.js'); const fs = require('fs'); -const TEST_URL = 'https://github.com/artilleryio/artillery/'; +const TEST_URL = 'https://www.artillery.io/'; const tags = getTestTags(['typescript:true']); let playwrightOutput; @@ -59,12 +59,21 @@ test('playwright typescript test works and reports data', async (t) => { //Assert: reports steps as histograms t.hasProp( summaries, - 'browser.step.go_to_artillery_repo', - 'should have reported step go_to_artillery_repo as histogram' + 'browser.step.go_to_artillery_io', + 'should have reported step go_to_artillery_io as histogram' ); t.ok( - Object.keys(summaries['browser.step.go_to_artillery_repo']).includes('p99'), - 'should have reported step go_to_artillery_repo as histogram with p99 metric' + Object.keys(summaries['browser.step.go_to_artillery_io']).includes('p99'), + 'should have reported step go_to_artillery_io as histogram with p99 metric' + ); + t.hasProp( + summaries, + 'browser.step.go_to_docs', + 'should have reported step go_to_docs as histogram' + ); + t.ok( + Object.keys(summaries['browser.step.go_to_docs']).includes('p99'), + 'should have reported step go_to_docs as histogram with p99 metric' ); //Assert: reports web vital metrics diff --git a/packages/artillery-engine-playwright/test/fixtures/processor.js b/packages/artillery-engine-playwright/test/fixtures/processor.js index 2271775f10..efb66ec4e1 100644 --- a/packages/artillery-engine-playwright/test/fixtures/processor.js +++ b/packages/artillery-engine-playwright/test/fixtures/processor.js @@ -1,16 +1,47 @@ const { expect } = require('@playwright/test'); +//this is due to occasional failures in CI due to known unresolved issue: https://github.com/microsoft/playwright/issues/13062 +const retryGoingToPage = async (page, url) => { + let retryCount = 0; + let error; + while (retryCount < 5) { + try { + await page.goto(url); + return; + } catch (err) { + console.log(`ERROR: page.goto in Playwright test - ${err.message}`); + console.log('Retrying...'); + error = err; + retryCount++; + } + } + throw new Error(`Failed to go to page ${url}: ${error}`); +}; + async function artilleryPlaywrightFunction(page, vuContext, events, test) { - await test.step('go_to_artillery_repo', async () => { - await page.goto(`${vuContext.vars.target}/`); - await expect(page.getByTestId('latest-commit')).toBeVisible(); + await test.step('go_to_artillery_io', async () => { + await retryGoingToPage(page, '/'); + + await expect(page.getByText('The Artillery Manifesto')).toBeVisible(); }); + + await test.step('go_to_docs', async () => { + await page + .getByLabel('Main navigation') + .getByRole('link', { name: 'Documentation' }) + .click(); + await expect(page).toHaveURL('/docs'); + await expect( + page.getByText("What's different about Artillery?") + ).toBeVisible(); + }); + events.emit('counter', 'custom_emitter', 1); } async function playwrightFunctionWithFailure(page, vuContext, events, test) { - await test.step('go_to_artillery_repo', async () => { - await page.goto(`${vuContext.vars.target}/`); + await test.step('go_to_artillery_io', async () => { + await retryGoingToPage(page, '/'); await expect(page.getByText('gremlins are here!')).toBeVisible(); }); events.emit('counter', 'custom_emitter', 1); diff --git a/packages/artillery-engine-playwright/test/fixtures/processor.ts b/packages/artillery-engine-playwright/test/fixtures/processor.ts index 1885e65901..3d7241ca85 100644 --- a/packages/artillery-engine-playwright/test/fixtures/processor.ts +++ b/packages/artillery-engine-playwright/test/fixtures/processor.ts @@ -1,14 +1,43 @@ import { expect, Page } from '@playwright/test'; +//this is due to occasional failures in CI due to known unresolved issue (ERR_NETWORK_CHANGED): https://github.com/microsoft/playwright/issues/13062 +const retryGoingToPage = async (page, url) => { + let retryCount = 0; + let error; + while (retryCount < 10) { + try { + await page.goto(url); + return; + } catch (err) { + console.log(`ERROR: page.goto in Playwright test - ${err.message}`); + console.log(`Retrying...`); + error = err; + retryCount++; + } + } + throw new Error(`Failed to go to page ${url}: ${error}`); +}; + export async function artilleryPlaywrightFunction( page: Page, vuContext, events, test ) { - await test.step('go_to_artillery_repo', async () => { - await page.goto(`${vuContext.vars.target}/`); - await expect(page.getByTestId('latest-commit')).toBeVisible(); + await test.step('go_to_artillery_io', async () => { + await retryGoingToPage(page, '/'); + await expect(page.getByText('The Artillery Manifesto')).toBeVisible(); + }); + + await test.step('go_to_docs', async () => { + await page + .getByLabel('Main navigation') + .getByRole('link', { name: 'Documentation' }) + .click(); + await expect(page).toHaveURL('/docs'); + await expect( + page.getByText("What's different about Artillery?") + ).toBeVisible(); }); } @@ -18,8 +47,8 @@ export async function playwrightFunctionWithFailure( events, test ) { - await test.step('go_to_artillery_repo', async () => { - await page.goto(`${vuContext.vars.target}/`); + await test.step('go_to_artillery_io', async () => { + await retryGoingToPage(page, '/'); await expect(page.getByText('gremlins are here!')).toBeVisible(); }); events.emit('counter', 'custom_emitter', 1); diff --git a/packages/artillery-engine-playwright/test/fixtures/pw-acceptance.yml b/packages/artillery-engine-playwright/test/fixtures/pw-acceptance.yml index ddcbf3531e..6ebc61982f 100644 --- a/packages/artillery-engine-playwright/test/fixtures/pw-acceptance.yml +++ b/packages/artillery-engine-playwright/test/fixtures/pw-acceptance.yml @@ -1,5 +1,5 @@ config: - target: "https://github.com/artilleryio/artillery" + target: "https://www.artillery.io/" phases: - duration: 3 arrivalRate: 1 diff --git a/packages/artillery-engine-playwright/test/index.test.js b/packages/artillery-engine-playwright/test/index.test.js index d24082d604..e7c7aed130 100644 --- a/packages/artillery-engine-playwright/test/index.test.js +++ b/packages/artillery-engine-playwright/test/index.test.js @@ -2,7 +2,7 @@ const { test, afterEach, beforeEach } = require('tap'); const { $ } = require('zx'); const fs = require('fs'); -const TEST_URL = 'https://github.com/artilleryio/artillery/'; +const TEST_URL = 'https://www.artillery.io/'; let playwrightOutput; beforeEach(() => { @@ -54,12 +54,21 @@ test('playwright js test works and reports data', async (t) => { //Assert: reports steps as histograms t.hasProp( summaries, - 'browser.step.go_to_artillery_repo', - 'should have reported step go_to_artillery_repo as histogram' + 'browser.step.go_to_artillery_io', + 'should have reported step go_to_artillery_io as histogram' ); t.ok( - Object.keys(summaries['browser.step.go_to_artillery_repo']).includes('p99'), - 'should have reported step go_to_artillery_repo as histogram with p99 metric' + Object.keys(summaries['browser.step.go_to_artillery_io']).includes('p99'), + 'should have reported step go_to_artillery_io as histogram with p99 metric' + ); + t.hasProp( + summaries, + 'browser.step.go_to_docs', + 'should have reported step go_to_docs as histogram' + ); + t.ok( + Object.keys(summaries['browser.step.go_to_docs']).includes('p99'), + 'should have reported step go_to_docs as histogram with p99 metric' ); //Assert: reports web vital metrics @@ -180,12 +189,22 @@ test('playwright typescript test works and reports data', async (t) => { //Assert: reports steps as histograms t.hasProp( summaries, - 'browser.step.go_to_artillery_repo', - 'should have reported step go_to_artillery_repo as histogram' + 'browser.step.go_to_artillery_io', + 'should have reported step go_to_artillery_io as histogram' + ); + t.ok( + Object.keys(summaries['browser.step.go_to_artillery_io']).includes('p99'), + 'should have reported step go_to_artillery_io as histogram with p99 metric' + ); + + t.hasProp( + summaries, + 'browser.step.go_to_docs', + 'should have reported step go_to_artillery_io as histogram' ); t.ok( - Object.keys(summaries['browser.step.go_to_artillery_repo']).includes('p99'), - 'should have reported step go_to_artillery_repo as histogram with p99 metric' + Object.keys(summaries['browser.step.go_to_docs']).includes('p99'), + 'should have reported step go_to_docs as histogram with p99 metric' ); //Assert: reports web vital metrics