From d8cda856a69bc946dba2fcc5846de06ec691a82c Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Wed, 7 Sep 2022 11:49:19 -0700 Subject: [PATCH 1/3] tests(devtools): sync e2e tests --- .../e2e/lighthouse/navigation_test.ts | 91 +++++++++++-------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts b/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts index 8a0f514b5a69..e5c93a40a7fb 100644 --- a/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts +++ b/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts @@ -10,41 +10,17 @@ import { clickStartButton, getAuditsBreakdown, navigateToLighthouseTab, + selectCategories, + selectMode, + setClearStorage, setLegacyNavigation, setThrottlingMethod, waitForResult, } from '../helpers/lighthouse-helpers.js'; -import {click, goToResource, waitFor, setDevToolsSettings, waitForElementWithTextContent} from '../../shared/helper.js'; - -import type {ElementHandle} from 'puppeteer'; // This test will fail (by default) in headful mode, as the target page never gets painted. // To resolve this when debugging, just make sure the target page is visible during the lighthouse run. -// TODO: update upstream. -async function navigateToLighthouseTab_2(path?: string): Promise> { - await click('#tab-lighthouse'); - // await waitForLighthousePanelContentLoaded(); - await waitFor('.view-container > .lighthouse'); - if (path) { - await goToResource(path); - } - - return waitFor('.lighthouse-start-view-fr'); -} - -async function setLegacyNavigation_2(enabled: boolean, textContext = 'Legacy navigation') { - const toolbarHandle = await waitFor('.lighthouse-settings-pane .toolbar'); - const label = await waitForElementWithTextContent(textContext, toolbarHandle); - await label.evaluate((label, enabled: boolean) => { - const rootNode = label.getRootNode() as ShadowRoot; - const checkboxId = label.getAttribute('for') as string; - const checkboxElem = rootNode.getElementById(checkboxId) as HTMLInputElement; - checkboxElem.checked = enabled; - checkboxElem.dispatchEvent(new Event('change')); // Need change event to update the backing setting. - }, enabled); -} - describe('Navigation', async function() { // The tests in this suite are particularly slow this.timeout(60_000); @@ -58,26 +34,30 @@ describe('Navigation', async function() { // TODO: Figure out why these are emitted in FR. expectError(/Protocol Error: the message with wrong session id/); expectError(/Protocol Error: the message with wrong session id/); + expectError(/Protocol Error: the message with wrong session id/); + expectError(/Protocol Error: the message with wrong session id/); + expectError(/Protocol Error: the message with wrong session id/); } }); it('successfully returns a Lighthouse report', async () => { - await setDevToolsSettings({language: 'en-XL'}); - await navigateToLighthouseTab_2('lighthouse/hello.html'); + await navigateToLighthouseTab('lighthouse/hello.html'); + + await setLegacyNavigation(mode === 'legacy'); - await setLegacyNavigation_2(mode === 'legacy', 'L̂éĝáĉý n̂áv̂íĝát̂íôń'); await clickStartButton(); const {lhr, artifacts, reportEl} = await waitForResult(); - // TODO: Reenable this for 10.0 - // 9.6.x is forked so Lighthouse ToT is still using 9.5.0 as the version. - // assert.strictEqual(lhr.lighthouseVersion, '9.6.2'); + assert.strictEqual(lhr.lighthouseVersion, '9.6.6'); assert.match(lhr.finalUrl, /^https:\/\/localhost:[0-9]+\/test\/e2e\/resources\/lighthouse\/hello.html/); assert.strictEqual(lhr.configSettings.throttlingMethod, 'simulate'); + assert.strictEqual(lhr.configSettings.disableStorageReset, false); + assert.strictEqual(lhr.configSettings.formFactor, 'mobile'); const {innerWidth, innerHeight, outerWidth, outerHeight, devicePixelRatio} = artifacts.ViewportDimensions; // This value can vary slightly, depending on the display. + // https://bugs.chromium.org/p/chromium/issues/detail?id=1346355 assert.approximately(innerHeight, 1742, 1); assert.strictEqual(innerWidth, 980); assert.strictEqual(outerWidth, 360); @@ -85,12 +65,13 @@ describe('Navigation', async function() { assert.strictEqual(devicePixelRatio, 3); const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr); - assert.strictEqual(auditResults.length, 150); + assert.strictEqual(auditResults.length, 152); assert.strictEqual(erroredAudits.length, 0); assert.deepStrictEqual(failedAudits.map(audit => audit.id), [ 'service-worker', 'viewport', 'installable-manifest', + 'apple-touch-icon', 'splash-screen', 'themed-omnibox', 'maskable-icon', @@ -105,9 +86,7 @@ describe('Navigation', async function() { const viewTraceText = await reportEl.$eval('.lh-button--trace', viewTraceEl => { return viewTraceEl.textContent; }); - assert.strictEqual(viewTraceText, 'V̂íêẃ Ôŕîǵîńâĺ T̂ŕâćê'); - - assert.strictEqual(lhr.i18n.rendererFormattedStrings.footerIssue, 'F̂íl̂é âń îśŝúê'); + assert.strictEqual(viewTraceText, 'View Original Trace'); }); it('successfully returns a Lighthouse report with DevTools throttling', async () => { @@ -115,19 +94,27 @@ describe('Navigation', async function() { await setThrottlingMethod('devtools'); await setLegacyNavigation(mode === 'legacy'); + await clickStartButton(); const {lhr, reportEl} = await waitForResult(); assert.strictEqual(lhr.configSettings.throttlingMethod, 'devtools'); - const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr); - assert.strictEqual(auditResults.length, 150); + // [crbug.com/1347220] DevTools throttling can force resources to load slow enough for these audits to fail sometimes. + const flakyAudits = [ + 'server-response-time', + 'render-blocking-resources', + ]; + + const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr, flakyAudits); + assert.strictEqual(auditResults.length, 152); assert.strictEqual(erroredAudits.length, 0); assert.deepStrictEqual(failedAudits.map(audit => audit.id), [ 'service-worker', 'viewport', 'installable-manifest', + 'apple-touch-icon', 'splash-screen', 'themed-omnibox', 'maskable-icon', @@ -144,6 +131,32 @@ describe('Navigation', async function() { }); assert.strictEqual(viewTraceText, 'View Trace'); }); + + it('successfully returns a Lighthouse report when settings changed', async () => { + await navigateToLighthouseTab('lighthouse/hello.html'); + + await setLegacyNavigation(mode === 'legacy'); + await setClearStorage(false); + await selectCategories(['performance', 'best-practices']); + await selectMode('desktop'); + + await clickStartButton(); + + const {lhr, artifacts} = await waitForResult(); + + const {innerWidth, innerHeight, devicePixelRatio} = artifacts.ViewportDimensions; + // TODO: Figure out why outerHeight can be different depending on OS + assert.strictEqual(innerHeight, 720); + assert.strictEqual(innerWidth, 1280); + assert.strictEqual(devicePixelRatio, 1); + + const {erroredAudits} = getAuditsBreakdown(lhr); + assert.strictEqual(erroredAudits.length, 0); + + assert.deepStrictEqual(Object.keys(lhr.categories), ['performance', 'best-practices']); + assert.strictEqual(lhr.configSettings.disableStorageReset, true); + assert.strictEqual(lhr.configSettings.formFactor, 'desktop'); + }); }); } }); From 07a6394910005bf77e519e50259e34679d190c8d Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Wed, 7 Sep 2022 12:08:24 -0700 Subject: [PATCH 2/3] ope --- .../e2e/lighthouse/navigation_test.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts b/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts index e5c93a40a7fb..a86b060561bd 100644 --- a/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts +++ b/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts @@ -5,6 +5,7 @@ import {assert} from 'chai'; import {expectError} from '../../conductor/events.js'; +import {setDevToolsSettings} from '../../shared/helper.js'; import {describe, it} from '../../shared/mocha-extensions.js'; import { clickStartButton, @@ -12,9 +13,9 @@ import { navigateToLighthouseTab, selectCategories, selectMode, - setClearStorage, setLegacyNavigation, setThrottlingMethod, + setToolbarCheckboxWithText, waitForResult, } from '../helpers/lighthouse-helpers.js'; @@ -133,16 +134,17 @@ describe('Navigation', async function() { }); it('successfully returns a Lighthouse report when settings changed', async () => { + await setDevToolsSettings({language: 'en-XL'}); await navigateToLighthouseTab('lighthouse/hello.html'); - await setLegacyNavigation(mode === 'legacy'); - await setClearStorage(false); + await setToolbarCheckboxWithText(mode === 'legacy', 'L̂éĝáĉý n̂áv̂íĝát̂íôń'); + await setToolbarCheckboxWithText(false, 'Ĉĺêár̂ śt̂ór̂áĝé'); await selectCategories(['performance', 'best-practices']); await selectMode('desktop'); await clickStartButton(); - const {lhr, artifacts} = await waitForResult(); + const {reportEl, lhr, artifacts} = await waitForResult(); const {innerWidth, innerHeight, devicePixelRatio} = artifacts.ViewportDimensions; // TODO: Figure out why outerHeight can be different depending on OS @@ -156,6 +158,13 @@ describe('Navigation', async function() { assert.deepStrictEqual(Object.keys(lhr.categories), ['performance', 'best-practices']); assert.strictEqual(lhr.configSettings.disableStorageReset, true); assert.strictEqual(lhr.configSettings.formFactor, 'desktop'); + + const viewTraceText = await reportEl.$eval('.lh-button--trace', viewTraceEl => { + return viewTraceEl.textContent; + }); + assert.strictEqual(viewTraceText, 'V̂íêẃ Ôŕîǵîńâĺ T̂ŕâćê'); + + assert.strictEqual(lhr.i18n.rendererFormattedStrings.footerIssue, 'F̂íl̂é âń îśŝúê'); }); }); } From 21a4710596ae14a14841d0ef8c1a7de0d21fbd69 Mon Sep 17 00:00:00 2001 From: Adam Raine Date: Wed, 7 Sep 2022 12:16:19 -0700 Subject: [PATCH 3/3] ope --- .../devtools-tests/e2e/lighthouse/navigation_test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts b/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts index a86b060561bd..9bcbb61de257 100644 --- a/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts +++ b/third-party/devtools-tests/e2e/lighthouse/navigation_test.ts @@ -50,7 +50,9 @@ describe('Navigation', async function() { const {lhr, artifacts, reportEl} = await waitForResult(); - assert.strictEqual(lhr.lighthouseVersion, '9.6.6'); + // TODO: Reenable this for 10.0 + // 9.6.x is forked so Lighthouse ToT is still using 9.5.0 as the version. + // assert.strictEqual(lhr.lighthouseVersion, '9.6.6'); assert.match(lhr.finalUrl, /^https:\/\/localhost:[0-9]+\/test\/e2e\/resources\/lighthouse\/hello.html/); assert.strictEqual(lhr.configSettings.throttlingMethod, 'simulate'); assert.strictEqual(lhr.configSettings.disableStorageReset, false); @@ -66,13 +68,12 @@ describe('Navigation', async function() { assert.strictEqual(devicePixelRatio, 3); const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr); - assert.strictEqual(auditResults.length, 152); + assert.strictEqual(auditResults.length, 150); assert.strictEqual(erroredAudits.length, 0); assert.deepStrictEqual(failedAudits.map(audit => audit.id), [ 'service-worker', 'viewport', 'installable-manifest', - 'apple-touch-icon', 'splash-screen', 'themed-omnibox', 'maskable-icon', @@ -109,13 +110,12 @@ describe('Navigation', async function() { ]; const {auditResults, erroredAudits, failedAudits} = getAuditsBreakdown(lhr, flakyAudits); - assert.strictEqual(auditResults.length, 152); + assert.strictEqual(auditResults.length, 150); assert.strictEqual(erroredAudits.length, 0); assert.deepStrictEqual(failedAudits.map(audit => audit.id), [ 'service-worker', 'viewport', 'installable-manifest', - 'apple-touch-icon', 'splash-screen', 'themed-omnibox', 'maskable-icon',