From eb2b32eb90c4f38d1111dab88f1e25ab17f3c58e Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 13:53:21 -0700 Subject: [PATCH 01/20] core(source-maps): workaround CORS for fetching maps --- .../test/fixtures/source-map/script.js.map | 8 + .../source-map/source-map-tester.html | 31 ++++ .../test/smokehouse/smoke-test-dfns.js | 5 + .../test/smokehouse/source-map/config.js | 26 +++ .../smokehouse/source-map/expectations.js | 39 +++++ lighthouse-core/gather/driver.js | 148 ++++++++++++++++++ .../gather/gatherers/source-maps.js | 31 +--- .../test/gather/gatherers/source-maps-test.js | 36 +++-- package.json | 2 +- yarn.lock | 8 +- 10 files changed, 288 insertions(+), 46 deletions(-) create mode 100644 lighthouse-cli/test/fixtures/source-map/script.js.map create mode 100644 lighthouse-cli/test/fixtures/source-map/source-map-tester.html create mode 100644 lighthouse-cli/test/smokehouse/source-map/config.js create mode 100644 lighthouse-cli/test/smokehouse/source-map/expectations.js diff --git a/lighthouse-cli/test/fixtures/source-map/script.js.map b/lighthouse-cli/test/fixtures/source-map/script.js.map new file mode 100644 index 000000000000..8bab5ab197de --- /dev/null +++ b/lighthouse-cli/test/fixtures/source-map/script.js.map @@ -0,0 +1,8 @@ +{ + "version": 3, + "file": "out.js", + "sourceRoot": "", + "sources": ["foo.js", "bar.js"], + "names": ["src", "maps", "are", "fun"], + "mappings": "AAgBC,SAAQ,CAAEA" +} diff --git a/lighthouse-cli/test/fixtures/source-map/source-map-tester.html b/lighthouse-cli/test/fixtures/source-map/source-map-tester.html new file mode 100644 index 000000000000..8f3f49766702 --- /dev/null +++ b/lighthouse-cli/test/fixtures/source-map/source-map-tester.html @@ -0,0 +1,31 @@ + + + + + + + + + Source maps tester + + + + + + + + map time map time! map time map time! 🎉 + + + \ No newline at end of file diff --git a/lighthouse-cli/test/smokehouse/smoke-test-dfns.js b/lighthouse-cli/test/smokehouse/smoke-test-dfns.js index c12b14fcdffe..a0b0df8595ee 100644 --- a/lighthouse-cli/test/smokehouse/smoke-test-dfns.js +++ b/lighthouse-cli/test/smokehouse/smoke-test-dfns.js @@ -79,6 +79,11 @@ const SMOKE_TEST_DFNS = [{ expectations: 'tricky-metrics/expectations.js', config: 'lighthouse-core/config/perf-config.js', batch: 'parallel-second', +}, { + id: 'source-maps', + expectations: 'source-map/expectations.js', + config: 'source-map/config.js', + batch: 'parallel-first', }]; /** diff --git a/lighthouse-cli/test/smokehouse/source-map/config.js b/lighthouse-cli/test/smokehouse/source-map/config.js new file mode 100644 index 000000000000..12ede17b3903 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/source-map/config.js @@ -0,0 +1,26 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * Config file for running source map smokehouse. + */ + +// source-maps currently isn't in the default config yet, so we make a new one with it. +// Also, no audits use source-maps yet, and at least one is required for a successful run, +// so `viewport` and its required gatherer `meta-elements` is used. + +/** @type {LH.Config.Json} */ +module.exports = { + passes: [{ + passName: 'defaultPass', + gatherers: [ + 'source-maps', + 'meta-elements', + ], + }], + audits: ['viewport'], +}; diff --git a/lighthouse-cli/test/smokehouse/source-map/expectations.js b/lighthouse-cli/test/smokehouse/source-map/expectations.js new file mode 100644 index 000000000000..e7aa49869d2b --- /dev/null +++ b/lighthouse-cli/test/smokehouse/source-map/expectations.js @@ -0,0 +1,39 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const fs = require('fs'); + +const mapPath = require.resolve('../../fixtures/source-map/script.js.map'); +const mapJson = fs.readFileSync(mapPath, 'utf-8'); +const map = JSON.parse(mapJson); + +/** + * Expected Lighthouse results for source maps. + */ +module.exports = [ + { + artifacts: { + SourceMaps: [ + { + scriptUrl: 'http://localhost:10200/source-map/source-map-tester.html', + sourceMapUrl: 'http://localhost:10200/source-map/script.js.map', + map, + }, + { + scriptUrl: 'http://localhost:10200/source-map/source-map-tester.html', + sourceMapUrl: 'http://localhost:10503/source-map/script.js.map', + map, + }, + ], + }, + lhr: { + requestedUrl: 'http://localhost:10200/source-map/source-map-tester.html', + finalUrl: 'http://localhost:10200/source-map/source-map-tester.html', + audits: {}, + }, + }, +]; diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js index 1d0cfea5de49..34185a3bf610 100644 --- a/lighthouse-core/gather/driver.js +++ b/lighthouse-core/gather/driver.js @@ -103,6 +103,11 @@ class Driver { * @private */ this._nextProtocolTimeout = DEFAULT_PROTOCOL_TIMEOUT; + + this._onRequestPaused = this._onRequestPaused.bind(this); + + /** @type {Map void>} */ + this._onRequestPausedHandlers = new Map(); } static get traceCategories() { @@ -1581,6 +1586,149 @@ class Driver { await this.sendCommand('Page.enable'); } + + /** + * The Fetch domain accepts patterns for controlling what requests are intercepted, but we + * enable the domain for all patterns and filter events at a lower level to support multiple + * concurrent usages. Reasons for this: + * + * 1) only one set of patterns may be applied for the entire domain. + * 2) every request that matches the patterns are paused and only resumes when certain Fetch + * commands are sent. So a listener of the `Fetch.requestPaused` event must either handle + * the requests it cares about, or explicitly allow them to continue. + * 3) if multiple commands to continue the same request are sent, protocol errors occur. + * + * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific + * urls to be intercepted via `driver.setOnRequestPausedHandler`. + */ + async enableRequestInterception() { + await this.sendCommand('Fetch.enable', { + patterns: [{requestStage: 'Request'}, {requestStage: 'Response'}], + }); + await this.on('Fetch.requestPaused', this._onRequestPaused); + } + + /** + * @param {string} url + * @param {(event: LH.Crdp.Fetch.RequestPausedEvent) => void} handler + */ + async setOnRequestPausedHandler(url, handler) { + this._onRequestPausedHandlers.set(url, handler); + } + + /** + * @param {LH.Crdp.Fetch.RequestPausedEvent} event + */ + async _onRequestPaused(event) { + const handler = this._onRequestPausedHandlers.get(event.request.url); + if (handler) { + await handler(event); + } else { + // Nothing cares about this URL, so continue. + await this.sendCommand('Fetch.continueRequest', {requestId: event.requestId}); + } + } + + async disableRequestInterception() { + await this.sendCommand('Fetch.disable'); + await this.off('Fetch.requestPaused', this._onRequestPaused); + this._onRequestPausedHandlers.clear(); + } + + /** + * Requires that `driver.enableRequestInterception` has been called. + * + * Fetches any resource in a way that circumvents CORS. + * + * @param {string} url + * @param {number} timeoutInMs + */ + async fetchArbitraryResource(url, timeoutInMs = 500) { + if (!this.isDomainEnabled('Fetch')) { + throw new Error('Fetch domain must be enabled to use fetchArbitraryResource'); + } + + /** @type {Promise} */ + const requestInterceptionPromise = new Promise((resolve, reject) => { + this.setOnRequestPausedHandler(url, async (event) => { + const {requestId, responseStatusCode} = event; + + // The first requestPaused event is for the request stage. Continue it. + if (!responseStatusCode) { + // Remove same-site cookies so we aren't buying stuff on Amazon. + const sameSiteCookies = await this.sendCommand('Network.getCookies', {urls: [url]}); + const sameSiteCookiesKeyValueSet = new Set(); + for (const cookie of sameSiteCookies.cookies) { + sameSiteCookiesKeyValueSet.add(cookie.name + '=' + cookie.value); + } + const strippedCookies = event.request.headers['Cookie'] + .split(';') + .filter(cookieKeyValue => { + return !sameSiteCookiesKeyValueSet.has(cookieKeyValue.trim()); + }) + .join('; '); + + this.sendCommand('Fetch.continueRequest', { + requestId, + headers: [{name: 'Cookie', value: strippedCookies}], + }); + return; + } + + // Now in the response stage, but the request failed. + if (!(responseStatusCode >= 200 && responseStatusCode < 300)) { + reject(new Error(`Invalid response status code: ${responseStatusCode}`)); + return; + } + + const responseBody = await this.sendCommand('Fetch.getResponseBody', {requestId}); + if (responseBody.base64Encoded) { + resolve(Buffer.from(responseBody.body, 'base64').toString()); + } else { + resolve(responseBody.body); + } + + // Fail the request (from the page's perspective) so that the iframe never loads. + this.sendCommand('Fetch.failRequest', {requestId, errorReason: 'Aborted'}); + }); + }); + + /** + * @param {string} src + */ + /* istanbul ignore next */ + function injectIframe(src) { + /** @type {HTMLIFrameElement} */ + const iframe = document.createElement('iframe'); + // Try really hard not to affect the page. + iframe.style.display = 'none'; + iframe.style.position = 'absolute'; + iframe.style.left = '10000px'; + iframe.style.visibility = 'hidden'; + iframe.src = src; + iframe.onload = iframe.onerror = () => { + iframe.remove(); + delete iframe.onload; + delete iframe.onerror; + }; + document.body.appendChild(iframe); + } + + await this.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`); + + /** @type {NodeJS.Timeout} */ + let timeoutHandle; + /** @type {Promise} */ + const timeoutPromise = new Promise((_, reject) => { + const errorMessage = 'Timed out fetching resource.'; + timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeoutInMs); + }); + + return Promise.race([ + timeoutPromise, + requestInterceptionPromise, + ]).finally(() => clearTimeout(timeoutHandle)); + } } module.exports = Driver; diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 17f7405af768..9a5575d3cab9 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -10,25 +10,6 @@ const Gatherer = require('./gatherer.js'); const URL = require('../../lib/url-shim.js'); -/** - * This function fetches source maps; it is careful not to parse the response as JSON, as it will - * just need to be serialized again over the protocol, and source maps can - * be huge. - * - * @param {string} url - * @return {Promise} - */ -/* istanbul ignore next */ -async function fetchSourceMap(url) { - // eslint-disable-next-line no-undef - const response = await fetch(url); - if (response.ok) { - return response.text(); - } else { - throw new Error(`Received status code ${response.status} for ${url}`); - } -} - /** * @fileoverview Gets JavaScript source maps. */ @@ -45,11 +26,9 @@ class SourceMaps extends Gatherer { * @param {string} sourceMapUrl * @return {Promise} */ - async fetchSourceMapInPage(driver, sourceMapUrl) { - driver.setNextProtocolTimeout(1500); + async fetchSourceMap(driver, sourceMapUrl) { /** @type {string} */ - const sourceMapJson = - await driver.evaluateAsync(`(${fetchSourceMap})(${JSON.stringify(sourceMapUrl)})`); + const sourceMapJson = await driver.fetchArbitraryResource(sourceMapUrl, 1500); return JSON.parse(sourceMapJson); } @@ -124,7 +103,7 @@ class SourceMaps extends Gatherer { try { const map = isSourceMapADataUri ? this.parseSourceMapFromDataUrl(rawSourceMapUrl) : - await this.fetchSourceMapInPage(driver, rawSourceMapUrl); + await this.fetchSourceMap(driver, rawSourceMapUrl); return { scriptUrl, sourceMapUrl, @@ -149,10 +128,10 @@ class SourceMaps extends Gatherer { driver.off('Debugger.scriptParsed', this.onScriptParsed); await driver.sendCommand('Debugger.disable'); + await driver.enableRequestInterception(); const eventProcessPromises = this._scriptParsedEvents .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event)); - - return Promise.all(eventProcessPromises); + return Promise.all(eventProcessPromises).finally(() => driver.disableRequestInterception()); } } diff --git a/lighthouse-core/test/gather/gatherers/source-maps-test.js b/lighthouse-core/test/gather/gatherers/source-maps-test.js index 5d809cbb7496..1df05d2430f4 100644 --- a/lighthouse-core/test/gather/gatherers/source-maps-test.js +++ b/lighthouse-core/test/gather/gatherers/source-maps-test.js @@ -34,11 +34,20 @@ describe('SourceMaps gatherer', () => { * @return {Promise} */ async function runSourceMaps(mapsAndEvents) { - const onMock = createMockOnFn(); + // pre-condition: should only define map or fetchError, not both. + for (const {map, fetchError} of mapsAndEvents) { + if (map && fetchError) { + throw new Error('should only define map or fetchError, not both.'); + } + } + const onMock = createMockOnFn(); const sendCommandMock = createMockSendCommandFn() .mockResponse('Debugger.enable', {}) - .mockResponse('Debugger.disable', {}); + .mockResponse('Debugger.disable', {}) + .mockResponse('Fetch.enable', {}) + .mockResponse('Fetch.disable', {}); + const fetchMock = jest.fn(); for (const {scriptParsedEvent, map, resolvedSourceMapUrl, fetchError} of mapsAndEvents) { onMock.mockEvent('protocolevent', { @@ -47,25 +56,21 @@ describe('SourceMaps gatherer', () => { }); if (scriptParsedEvent.sourceMapURL.startsWith('data:')) { - // Only the source maps that need to be fetched use the `evaluateAsync` code path. + // Only the source maps that need to be fetched use the `fetchMock` code path. continue; } - if (map && fetchError) { - throw new Error('should only define map or fetchError, not both.'); - } + fetchMock.mockImplementationOnce(async (sourceMapUrl) => { + // Check that the source map url was resolved correctly. + if (resolvedSourceMapUrl) { + expect(sourceMapUrl).toBe(resolvedSourceMapUrl); + } - sendCommandMock.mockResponse('Runtime.evaluate', ({expression}) => { - // Check that the source map url was resolved correctly. It'll be somewhere - // in the code sent to Runtime.evaluate. - if (resolvedSourceMapUrl && !expression.includes(resolvedSourceMapUrl)) { - throw new Error(`did not request expected url: ${resolvedSourceMapUrl}`); + if (fetchError) { + throw Object.assign(new Error(), {message: fetchError, __failedInBrowser: true}); } - const value = fetchError ? - Object.assign(new Error(), {message: fetchError, __failedInBrowser: true}) : - map; - return {result: {value}}; + return map; }); } const connectionStub = new Connection(); @@ -73,6 +78,7 @@ describe('SourceMaps gatherer', () => { connectionStub.on = onMock; const driver = new Driver(connectionStub); + driver.fetchArbitraryResource = fetchMock; const sourceMaps = new SourceMaps(); await sourceMaps.beforePass({driver}); diff --git a/package.json b/package.json index 5636c4f8c08f..397feb4c444e 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,7 @@ "cpy": "^7.0.1", "csv-validator": "^0.0.3", "cz-customizable": "^5.2.0", - "devtools-protocol": "0.0.588129", + "devtools-protocol": "0.0.680180", "eslint": "^4.19.1", "eslint-config-google": "^0.9.1", "eslint-plugin-local-rules": "0.1.0", diff --git a/yarn.lock b/yarn.lock index 6d1d187a8cd4..3481c7b9a1ee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2617,10 +2617,10 @@ detective@^5.0.2: defined "^1.0.0" minimist "^1.1.1" -devtools-protocol@0.0.588129: - version "0.0.588129" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.588129.tgz#b93d05b01679471639f2882077f77d4e96eecc18" - integrity sha512-5ID10jYNewQpBFo1IhKtYkH1b+EF9DmrnWidSdAnBq8hdJyZKxHvjHPbwTlWeqhzVzUKHSdn5EUj9BZq31U+QQ== +devtools-protocol@0.0.680180: + version "0.0.680180" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.680180.tgz#778aa24c8b55be267193e4cb6e9a949600258574" + integrity sha512-yd4VuVflxW3Bgb6qvZpDH+/SCXzkIZTkL5qO9t1i1xoRp5/3/XZMPvyZG6PWGmVKUHTZDom0Qs0dCmcb9UOCmw== diff-sequences@^24.3.0: version "24.3.0" From 637f8f438527204210d4ecd106dd01cc236314ed Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 14:34:44 -0700 Subject: [PATCH 02/20] changes --- lighthouse-cli/test/fixtures/source-map/source-map-tester.html | 1 - lighthouse-core/gather/driver.js | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-cli/test/fixtures/source-map/source-map-tester.html b/lighthouse-cli/test/fixtures/source-map/source-map-tester.html index 8f3f49766702..515607c646eb 100644 --- a/lighthouse-cli/test/fixtures/source-map/source-map-tester.html +++ b/lighthouse-cli/test/fixtures/source-map/source-map-tester.html @@ -15,7 +15,6 @@ diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js index 34185a3bf610..dde0fe62d4b0 100644 --- a/lighthouse-core/gather/driver.js +++ b/lighthouse-core/gather/driver.js @@ -1642,6 +1642,7 @@ class Driver { * * @param {string} url * @param {number} timeoutInMs + * @return {Promise} */ async fetchArbitraryResource(url, timeoutInMs = 500) { if (!this.isDomainEnabled('Fetch')) { From f20e927639a627f98150458a01cd1adee11c0ff3 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 15:36:44 -0700 Subject: [PATCH 03/20] fetcher.js --- lighthouse-core/gather/driver.js | 150 +--------------- lighthouse-core/gather/fetcher.js | 168 ++++++++++++++++++ .../gather/gatherers/source-maps.js | 6 +- .../test/gather/gatherers/source-maps-test.js | 2 +- 4 files changed, 174 insertions(+), 152 deletions(-) create mode 100644 lighthouse-core/gather/fetcher.js diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js index dde0fe62d4b0..c07a12efb8f7 100644 --- a/lighthouse-core/gather/driver.js +++ b/lighthouse-core/gather/driver.js @@ -5,6 +5,7 @@ */ 'use strict'; +const Fetcher = require('./fetcher.js'); const NetworkRecorder = require('../lib/network-recorder.js'); const emulation = require('../lib/emulation.js'); const Element = require('../lib/element.js'); @@ -104,10 +105,7 @@ class Driver { */ this._nextProtocolTimeout = DEFAULT_PROTOCOL_TIMEOUT; - this._onRequestPaused = this._onRequestPaused.bind(this); - - /** @type {Map void>} */ - this._onRequestPausedHandlers = new Map(); + this.fetcher = new Fetcher(this); } static get traceCategories() { @@ -1586,150 +1584,6 @@ class Driver { await this.sendCommand('Page.enable'); } - - /** - * The Fetch domain accepts patterns for controlling what requests are intercepted, but we - * enable the domain for all patterns and filter events at a lower level to support multiple - * concurrent usages. Reasons for this: - * - * 1) only one set of patterns may be applied for the entire domain. - * 2) every request that matches the patterns are paused and only resumes when certain Fetch - * commands are sent. So a listener of the `Fetch.requestPaused` event must either handle - * the requests it cares about, or explicitly allow them to continue. - * 3) if multiple commands to continue the same request are sent, protocol errors occur. - * - * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific - * urls to be intercepted via `driver.setOnRequestPausedHandler`. - */ - async enableRequestInterception() { - await this.sendCommand('Fetch.enable', { - patterns: [{requestStage: 'Request'}, {requestStage: 'Response'}], - }); - await this.on('Fetch.requestPaused', this._onRequestPaused); - } - - /** - * @param {string} url - * @param {(event: LH.Crdp.Fetch.RequestPausedEvent) => void} handler - */ - async setOnRequestPausedHandler(url, handler) { - this._onRequestPausedHandlers.set(url, handler); - } - - /** - * @param {LH.Crdp.Fetch.RequestPausedEvent} event - */ - async _onRequestPaused(event) { - const handler = this._onRequestPausedHandlers.get(event.request.url); - if (handler) { - await handler(event); - } else { - // Nothing cares about this URL, so continue. - await this.sendCommand('Fetch.continueRequest', {requestId: event.requestId}); - } - } - - async disableRequestInterception() { - await this.sendCommand('Fetch.disable'); - await this.off('Fetch.requestPaused', this._onRequestPaused); - this._onRequestPausedHandlers.clear(); - } - - /** - * Requires that `driver.enableRequestInterception` has been called. - * - * Fetches any resource in a way that circumvents CORS. - * - * @param {string} url - * @param {number} timeoutInMs - * @return {Promise} - */ - async fetchArbitraryResource(url, timeoutInMs = 500) { - if (!this.isDomainEnabled('Fetch')) { - throw new Error('Fetch domain must be enabled to use fetchArbitraryResource'); - } - - /** @type {Promise} */ - const requestInterceptionPromise = new Promise((resolve, reject) => { - this.setOnRequestPausedHandler(url, async (event) => { - const {requestId, responseStatusCode} = event; - - // The first requestPaused event is for the request stage. Continue it. - if (!responseStatusCode) { - // Remove same-site cookies so we aren't buying stuff on Amazon. - const sameSiteCookies = await this.sendCommand('Network.getCookies', {urls: [url]}); - const sameSiteCookiesKeyValueSet = new Set(); - for (const cookie of sameSiteCookies.cookies) { - sameSiteCookiesKeyValueSet.add(cookie.name + '=' + cookie.value); - } - const strippedCookies = event.request.headers['Cookie'] - .split(';') - .filter(cookieKeyValue => { - return !sameSiteCookiesKeyValueSet.has(cookieKeyValue.trim()); - }) - .join('; '); - - this.sendCommand('Fetch.continueRequest', { - requestId, - headers: [{name: 'Cookie', value: strippedCookies}], - }); - return; - } - - // Now in the response stage, but the request failed. - if (!(responseStatusCode >= 200 && responseStatusCode < 300)) { - reject(new Error(`Invalid response status code: ${responseStatusCode}`)); - return; - } - - const responseBody = await this.sendCommand('Fetch.getResponseBody', {requestId}); - if (responseBody.base64Encoded) { - resolve(Buffer.from(responseBody.body, 'base64').toString()); - } else { - resolve(responseBody.body); - } - - // Fail the request (from the page's perspective) so that the iframe never loads. - this.sendCommand('Fetch.failRequest', {requestId, errorReason: 'Aborted'}); - }); - }); - - /** - * @param {string} src - */ - /* istanbul ignore next */ - function injectIframe(src) { - /** @type {HTMLIFrameElement} */ - const iframe = document.createElement('iframe'); - // Try really hard not to affect the page. - iframe.style.display = 'none'; - iframe.style.position = 'absolute'; - iframe.style.left = '10000px'; - iframe.style.visibility = 'hidden'; - iframe.src = src; - iframe.onload = iframe.onerror = () => { - iframe.remove(); - delete iframe.onload; - delete iframe.onerror; - }; - document.body.appendChild(iframe); - } - - await this.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`); - - /** @type {NodeJS.Timeout} */ - let timeoutHandle; - /** @type {Promise} */ - const timeoutPromise = new Promise((_, reject) => { - const errorMessage = 'Timed out fetching resource.'; - timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeoutInMs); - }); - - return Promise.race([ - timeoutPromise, - requestInterceptionPromise, - ]).finally(() => clearTimeout(timeoutHandle)); - } } module.exports = Driver; diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js new file mode 100644 index 000000000000..37047c62747b --- /dev/null +++ b/lighthouse-core/gather/fetcher.js @@ -0,0 +1,168 @@ +/** + * @license Copyright 2016 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +class Fetcher { + /** + * @param {import('./driver.js')} driver + */ + constructor(driver) { + this.driver = driver; + /** @type {Map void>} */ + this._onRequestPausedHandlers = new Map(); + this._onRequestPaused = this._onRequestPaused.bind(this); + } + + /** + * The Fetch domain accepts patterns for controlling what requests are intercepted, but we + * enable the domain for all patterns and filter events at a lower level to support multiple + * concurrent usages. Reasons for this: + * + * 1) only one set of patterns may be applied for the entire domain. + * 2) every request that matches the patterns are paused and only resumes when certain Fetch + * commands are sent. So a listener of the `Fetch.requestPaused` event must either handle + * the requests it cares about, or explicitly allow them to continue. + * 3) if multiple commands to continue the same request are sent, protocol errors occur. + * + * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific + * urls to be intercepted via `driver.setOnRequestPausedHandler`. + */ + async enableRequestInterception() { + await this.driver.sendCommand('Fetch.enable', { + patterns: [{requestStage: 'Request'}, {requestStage: 'Response'}], + }); + await this.driver.on('Fetch.requestPaused', this._onRequestPaused); + } + + /** + * @param {string} url + * @param {(event: LH.Crdp.Fetch.RequestPausedEvent) => void} handler + */ + async setOnRequestPausedHandler(url, handler) { + this._onRequestPausedHandlers.set(url, handler); + } + + /** + * @param {LH.Crdp.Fetch.RequestPausedEvent} event + */ + async _onRequestPaused(event) { + const handler = this._onRequestPausedHandlers.get(event.request.url); + if (handler) { + await handler(event); + } else { + // Nothing cares about this URL, so continue. + await this.driver.sendCommand('Fetch.continueRequest', {requestId: event.requestId}); + } + } + + async disableRequestInterception() { + await this.driver.sendCommand('Fetch.disable'); + await this.driver.off('Fetch.requestPaused', this._onRequestPaused); + this._onRequestPausedHandlers.clear(); + } + + /** + * Requires that `driver.enableRequestInterception` has been called. + * + * Fetches any resource in a way that circumvents CORS. + * + * @param {string} url + * @param {number} timeoutInMs + * @return {Promise} + */ + async fetchResource(url, timeoutInMs = 500) { + if (!this.driver.isDomainEnabled('Fetch')) { + throw new Error('Must call `enableRequestInterception` before using fetchResource'); + } + + /** @type {Promise} */ + const requestInterceptionPromise = new Promise((resolve, reject) => { + this.setOnRequestPausedHandler(url, async (event) => { + const {requestId, responseStatusCode} = event; + + // The first requestPaused event is for the request stage. Continue it. + if (!responseStatusCode) { + // Remove same-site cookies so we aren't buying stuff on Amazon. + const headers = []; + if (event.request.headers['Cookie']) { + const sameSiteCookies = await this.driver.sendCommand('Network.getCookies', {urls: [url]}); + const sameSiteCookiesKeyValueSet = new Set(); + for (const cookie of sameSiteCookies.cookies) { + sameSiteCookiesKeyValueSet.add(cookie.name + '=' + cookie.value); + } + const strippedCookies = event.request.headers['Cookie'] + .split(';') + .filter(cookieKeyValue => { + return !sameSiteCookiesKeyValueSet.has(cookieKeyValue.trim()); + }) + .join('; '); + headers.push({name: 'Cookie', value: strippedCookies}); + } + + this.driver.sendCommand('Fetch.continueRequest', { + requestId, + headers, + }); + return; + } + + // Now in the response stage, but the request failed. + if (!(responseStatusCode >= 200 && responseStatusCode < 300)) { + reject(new Error(`Invalid response status code: ${responseStatusCode}`)); + return; + } + + const responseBody = await this.driver.sendCommand('Fetch.getResponseBody', {requestId}); + if (responseBody.base64Encoded) { + resolve(Buffer.from(responseBody.body, 'base64').toString()); + } else { + resolve(responseBody.body); + } + + // Fail the request (from the page's perspective) so that the iframe never loads. + this.driver.sendCommand('Fetch.failRequest', {requestId, errorReason: 'Aborted'}); + }); + }); + + /** + * @param {string} src + */ + /* istanbul ignore next */ + function injectIframe(src) { + /** @type {HTMLIFrameElement} */ + const iframe = document.createElement('iframe'); + // Try really hard not to affect the page. + iframe.style.display = 'none'; + iframe.style.position = 'absolute'; + iframe.style.left = '10000px'; + iframe.style.visibility = 'hidden'; + iframe.src = src; + iframe.onload = iframe.onerror = () => { + iframe.remove(); + delete iframe.onload; + delete iframe.onerror; + }; + document.body.appendChild(iframe); + } + + await this.driver.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`); + + /** @type {NodeJS.Timeout} */ + let timeoutHandle; + /** @type {Promise} */ + const timeoutPromise = new Promise((_, reject) => { + const errorMessage = 'Timed out fetching resource.'; + timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeoutInMs); + }); + + return Promise.race([ + timeoutPromise, + requestInterceptionPromise, + ]).finally(() => clearTimeout(timeoutHandle)); + } +} + +module.exports = Fetcher; diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 9a5575d3cab9..6e1901c617bb 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -28,7 +28,7 @@ class SourceMaps extends Gatherer { */ async fetchSourceMap(driver, sourceMapUrl) { /** @type {string} */ - const sourceMapJson = await driver.fetchArbitraryResource(sourceMapUrl, 1500); + const sourceMapJson = await driver.fetcher.fetchResource(sourceMapUrl, 1500); return JSON.parse(sourceMapJson); } @@ -128,10 +128,10 @@ class SourceMaps extends Gatherer { driver.off('Debugger.scriptParsed', this.onScriptParsed); await driver.sendCommand('Debugger.disable'); - await driver.enableRequestInterception(); + await driver.fetcher.enableRequestInterception(); const eventProcessPromises = this._scriptParsedEvents .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event)); - return Promise.all(eventProcessPromises).finally(() => driver.disableRequestInterception()); + return Promise.all(eventProcessPromises).finally(() => driver.fetcher.disableRequestInterception()); } } diff --git a/lighthouse-core/test/gather/gatherers/source-maps-test.js b/lighthouse-core/test/gather/gatherers/source-maps-test.js index 1df05d2430f4..5a77974692a3 100644 --- a/lighthouse-core/test/gather/gatherers/source-maps-test.js +++ b/lighthouse-core/test/gather/gatherers/source-maps-test.js @@ -78,7 +78,7 @@ describe('SourceMaps gatherer', () => { connectionStub.on = onMock; const driver = new Driver(connectionStub); - driver.fetchArbitraryResource = fetchMock; + driver.fetcher.fetchResource = fetchMock; const sourceMaps = new SourceMaps(); await sourceMaps.beforePass({driver}); From 6fcfd563c9d864d647866ba99c700cc22b40ecf0 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 15:37:56 -0700 Subject: [PATCH 04/20] top, isoloation --- lighthouse-core/gather/fetcher.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 37047c62747b..00a2045d77c2 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -137,7 +137,8 @@ class Fetcher { // Try really hard not to affect the page. iframe.style.display = 'none'; iframe.style.position = 'absolute'; - iframe.style.left = '10000px'; + iframe.style.top = '-1000px'; + iframe.style.left = '-1000px'; iframe.style.visibility = 'hidden'; iframe.src = src; iframe.onload = iframe.onerror = () => { @@ -148,7 +149,7 @@ class Fetcher { document.body.appendChild(iframe); } - await this.driver.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`); + await this.driver.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`, {useIsolation: true}); /** @type {NodeJS.Timeout} */ let timeoutHandle; From 61d0c2f9bed426f0224e5980480dd14f24291438 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 15:41:46 -0700 Subject: [PATCH 05/20] more dum dum styles --- lighthouse-core/gather/fetcher.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 00a2045d77c2..1390bd0eb36d 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -136,10 +136,12 @@ class Fetcher { const iframe = document.createElement('iframe'); // Try really hard not to affect the page. iframe.style.display = 'none'; + iframe.style.visibility = 'hidden'; iframe.style.position = 'absolute'; iframe.style.top = '-1000px'; iframe.style.left = '-1000px'; - iframe.style.visibility = 'hidden'; + iframe.style.width = '1px'; + iframe.style.height = '1px'; iframe.src = src; iframe.onload = iframe.onerror = () => { iframe.remove(); From 457f8c9bbda57b69fee6aa1becf9d1bdd1afcca9 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 25 Jul 2019 15:46:06 -0700 Subject: [PATCH 06/20] samesite --- lighthouse-core/gather/fetcher.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 1390bd0eb36d..b1728eec6ba7 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -88,10 +88,12 @@ class Fetcher { // Remove same-site cookies so we aren't buying stuff on Amazon. const headers = []; if (event.request.headers['Cookie']) { - const sameSiteCookies = await this.driver.sendCommand('Network.getCookies', {urls: [url]}); + const {cookies} = await this.driver.sendCommand('Network.getCookies', {urls: [url]}); const sameSiteCookiesKeyValueSet = new Set(); - for (const cookie of sameSiteCookies.cookies) { - sameSiteCookiesKeyValueSet.add(cookie.name + '=' + cookie.value); + for (const cookie of cookies) { + if (cookie.sameSite !== 'None') { + sameSiteCookiesKeyValueSet.add(cookie.name + '=' + cookie.value); + } } const strippedCookies = event.request.headers['Cookie'] .split(';') From f7a7652f1bae6fa3f347d77763542c9468b1b0e7 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Fri, 27 Sep 2019 22:48:30 -0700 Subject: [PATCH 07/20] Update lighthouse-core/gather/fetcher.js --- lighthouse-core/gather/fetcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index b1728eec6ba7..67070c9f982d 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -1,5 +1,5 @@ /** - * @license Copyright 2016 Google Inc. All Rights Reserved. + * @license Copyright 2019 Google Inc. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ From 1f3ec5d948de98740328b9639326022f6cdf17e5 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 26 Nov 2019 17:25:18 -0800 Subject: [PATCH 08/20] lint --- lighthouse-core/gather/fetcher.js | 6 +++++- lighthouse-core/gather/gatherers/source-maps.js | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 67070c9f982d..04ae543b672e 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -5,6 +5,8 @@ */ 'use strict'; +/* global document */ + class Fetcher { /** * @param {import('./driver.js')} driver @@ -153,7 +155,9 @@ class Fetcher { document.body.appendChild(iframe); } - await this.driver.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`, {useIsolation: true}); + await this.driver.evaluateAsync(`${injectIframe}(${JSON.stringify(url)})`, { + useIsolation: true, + }); /** @type {NodeJS.Timeout} */ let timeoutHandle; diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 6e1901c617bb..4dd0f97c5338 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -131,7 +131,8 @@ class SourceMaps extends Gatherer { await driver.fetcher.enableRequestInterception(); const eventProcessPromises = this._scriptParsedEvents .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event)); - return Promise.all(eventProcessPromises).finally(() => driver.fetcher.disableRequestInterception()); + return Promise.all(eventProcessPromises) + .finally(() => driver.fetcher.disableRequestInterception()); } } From 496ca2c6c4d353000c233904e2e9cdace8aa65db Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Tue, 26 Nov 2019 18:03:25 -0800 Subject: [PATCH 09/20] del --- .../test/smokehouse/smoke-test-dfns.js | 138 ------------------ 1 file changed, 138 deletions(-) delete mode 100644 lighthouse-cli/test/smokehouse/smoke-test-dfns.js diff --git a/lighthouse-cli/test/smokehouse/smoke-test-dfns.js b/lighthouse-cli/test/smokehouse/smoke-test-dfns.js deleted file mode 100644 index a0b0df8595ee..000000000000 --- a/lighthouse-cli/test/smokehouse/smoke-test-dfns.js +++ /dev/null @@ -1,138 +0,0 @@ -/** - * @license Copyright 2019 Google Inc. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - */ -'use strict'; - -const path = require('path'); -const smokehouseDir = 'lighthouse-cli/test/smokehouse/'; - -/** @type {Array} */ -const SMOKE_TEST_DFNS = [{ - id: 'a11y', - config: smokehouseDir + 'a11y/a11y-config.js', - expectations: 'a11y/expectations.js', - batch: 'parallel-first', -}, { - id: 'errors', - expectations: smokehouseDir + 'error-expectations.js', - config: smokehouseDir + 'error-config.js', - batch: 'errors', -}, { - id: 'oopif', - expectations: smokehouseDir + 'oopif-expectations.js', - config: smokehouseDir + 'oopif-config.js', - batch: 'parallel-first', -}, { - id: 'pwa', - expectations: smokehouseDir + 'pwa-expectations.js', - config: smokehouseDir + 'pwa-config.js', - batch: 'parallel-second', -}, { - id: 'pwa2', - expectations: smokehouseDir + 'pwa2-expectations.js', - config: smokehouseDir + 'pwa-config.js', - batch: 'parallel-second', -}, { - id: 'pwa3', - expectations: smokehouseDir + 'pwa3-expectations.js', - config: smokehouseDir + 'pwa-config.js', - batch: 'parallel-first', -}, { - id: 'dbw', - expectations: 'dobetterweb/dbw-expectations.js', - config: smokehouseDir + 'dbw-config.js', - batch: 'parallel-second', -}, { - id: 'redirects', - expectations: 'redirects/expectations.js', - config: smokehouseDir + 'redirects-config.js', - batch: 'parallel-first', -}, { - id: 'seo', - expectations: 'seo/expectations.js', - config: smokehouseDir + 'seo-config.js', - batch: 'parallel-first', -}, { - id: 'offline', - expectations: 'offline-local/offline-expectations.js', - config: smokehouseDir + 'offline-config.js', - batch: 'offline', -}, { - id: 'byte', - expectations: 'byte-efficiency/expectations.js', - config: smokehouseDir + 'byte-config.js', - batch: 'perf-opportunity', -}, { - id: 'perf', - expectations: 'perf/expectations.js', - config: 'perf/perf-config.js', - batch: 'perf-metric', -}, { - id: 'lantern', - expectations: 'perf/lantern-expectations.js', - config: smokehouseDir + 'lantern-config.js', - batch: 'parallel-first', -}, { - id: 'metrics', - expectations: 'tricky-metrics/expectations.js', - config: 'lighthouse-core/config/perf-config.js', - batch: 'parallel-second', -}, { - id: 'source-maps', - expectations: 'source-map/expectations.js', - config: 'source-map/config.js', - batch: 'parallel-first', -}]; - -/** - * Attempt to resolve a path relative to the smokehouse folder. - * If this fails, attempts to locate the path - * relative to the project root. - * @param {string} payloadPath - * @return {string} - */ -function resolveLocalOrProjectRoot(payloadPath) { - let resolved; - try { - resolved = require.resolve(__dirname + '/' + payloadPath); - } catch (e) { - const cwdPath = path.resolve(__dirname + '/../../../', payloadPath); - resolved = require.resolve(cwdPath); - } - - return resolved; -} - -/** - * @param {string} configPath - * @return {LH.Config.Json} - */ -function loadConfig(configPath) { - return require(configPath); -} - -/** - * @param {string} expectationsPath - * @return {Smokehouse.ExpectedRunnerResult[]} - */ -function loadExpectations(expectationsPath) { - return require(expectationsPath); -} - -function getSmokeTests() { - return SMOKE_TEST_DFNS.map(smokeTestDfn => { - return { - id: smokeTestDfn.id, - config: loadConfig(resolveLocalOrProjectRoot(smokeTestDfn.config)), - expectations: loadExpectations(resolveLocalOrProjectRoot(smokeTestDfn.expectations)), - batch: smokeTestDfn.batch, - }; - }); -} - -module.exports = { - SMOKE_TEST_DFNS, - getSmokeTests, -}; From 6687cf73440b64fc19287c48d883741d9de57eac Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Dec 2019 13:59:32 -0800 Subject: [PATCH 10/20] fix race condition --- lighthouse-core/gather/fetcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 04ae543b672e..533db2a20cde 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -61,8 +61,8 @@ class Fetcher { } async disableRequestInterception() { - await this.driver.sendCommand('Fetch.disable'); await this.driver.off('Fetch.requestPaused', this._onRequestPaused); + await this.driver.sendCommand('Fetch.disable'); this._onRequestPausedHandlers.clear(); } From 087cbc050425c96aceeafe7e1b6afda238d12939 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 27 Feb 2020 15:46:20 -0800 Subject: [PATCH 11/20] its 2020 yall --- lighthouse-core/gather/fetcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 533db2a20cde..25b4b8ead9d6 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -1,5 +1,5 @@ /** - * @license Copyright 2019 Google Inc. All Rights Reserved. + * @license Copyright 2020 Google Inc. All Rights Reserved. * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ From 586dbf8d67f222ec42b991c2513676950dc35f75 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 27 Feb 2020 16:15:46 -0800 Subject: [PATCH 12/20] update --- lighthouse-core/gather/fetcher.js | 12 +++++++++--- lighthouse-core/gather/gatherers/source-maps.js | 2 +- .../test/gather/gatherers/source-maps-test.js | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index 25b4b8ead9d6..a16fd38504a1 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -5,6 +5,12 @@ */ 'use strict'; +/** + * @fileoverview Fetcher is a utility for making requests within the context of the page. + * Requests can circumvent CORS, and so are good for fetching source maps that may be hosted + * on a different origin. + */ + /* global document */ class Fetcher { @@ -72,10 +78,10 @@ class Fetcher { * Fetches any resource in a way that circumvents CORS. * * @param {string} url - * @param {number} timeoutInMs + * @param {{timeout: number}} options timeout is in ms * @return {Promise} */ - async fetchResource(url, timeoutInMs = 500) { + async fetchResource(url, {timeout = 500}) { if (!this.driver.isDomainEnabled('Fetch')) { throw new Error('Must call `enableRequestInterception` before using fetchResource'); } @@ -164,7 +170,7 @@ class Fetcher { /** @type {Promise} */ const timeoutPromise = new Promise((_, reject) => { const errorMessage = 'Timed out fetching resource.'; - timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeoutInMs); + timeoutHandle = setTimeout(() => reject(new Error(errorMessage)), timeout); }); return Promise.race([ diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 7999ed05251d..3e43af60499f 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -28,7 +28,7 @@ class SourceMaps extends Gatherer { */ async fetchSourceMap(driver, sourceMapUrl) { /** @type {string} */ - const sourceMapJson = await driver.fetcher.fetchResource(sourceMapUrl, 1500); + const sourceMapJson = await driver.fetcher.fetchResource(sourceMapUrl, {timeout: 1500}); return JSON.parse(sourceMapJson); } diff --git a/lighthouse-core/test/gather/gatherers/source-maps-test.js b/lighthouse-core/test/gather/gatherers/source-maps-test.js index 5a77974692a3..b0998df422d2 100644 --- a/lighthouse-core/test/gather/gatherers/source-maps-test.js +++ b/lighthouse-core/test/gather/gatherers/source-maps-test.js @@ -67,7 +67,7 @@ describe('SourceMaps gatherer', () => { } if (fetchError) { - throw Object.assign(new Error(), {message: fetchError, __failedInBrowser: true}); + throw new Error(fetchError); } return map; From d66ed67cc4ae0581f22e5a427e57726529a2b245 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 27 Feb 2020 17:12:10 -0800 Subject: [PATCH 13/20] no cookies for u --- lighthouse-core/gather/fetcher.js | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index a16fd38504a1..a0ab626de002 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -93,23 +93,11 @@ class Fetcher { // The first requestPaused event is for the request stage. Continue it. if (!responseStatusCode) { - // Remove same-site cookies so we aren't buying stuff on Amazon. + // Remove cookies so we aren't buying stuff on Amazon. const headers = []; - if (event.request.headers['Cookie']) { - const {cookies} = await this.driver.sendCommand('Network.getCookies', {urls: [url]}); - const sameSiteCookiesKeyValueSet = new Set(); - for (const cookie of cookies) { - if (cookie.sameSite !== 'None') { - sameSiteCookiesKeyValueSet.add(cookie.name + '=' + cookie.value); - } - } - const strippedCookies = event.request.headers['Cookie'] - .split(';') - .filter(cookieKeyValue => { - return !sameSiteCookiesKeyValueSet.has(cookieKeyValue.trim()); - }) - .join('; '); - headers.push({name: 'Cookie', value: strippedCookies}); + for (const [key, value] of Object.entries(event.request.headers)) { + if (key === 'Cookie') return; + headers.push({name: key, value}); } this.driver.sendCommand('Fetch.continueRequest', { From 800c9e5b3542302a05106dafe265a4f52e5ad5a9 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Mar 2020 10:51:48 -0800 Subject: [PATCH 14/20] private --- lighthouse-core/gather/fetcher.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index a0ab626de002..cb51bdbad188 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -22,6 +22,7 @@ class Fetcher { /** @type {Map void>} */ this._onRequestPausedHandlers = new Map(); this._onRequestPaused = this._onRequestPaused.bind(this); + this._enabled = false; } /** @@ -36,20 +37,32 @@ class Fetcher { * 3) if multiple commands to continue the same request are sent, protocol errors occur. * * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific - * urls to be intercepted via `driver.setOnRequestPausedHandler`. + * urls to be intercepted via `driver._setOnRequestPausedHandler`. */ async enableRequestInterception() { + if (this._enabled) return; + + this._enabled = true; await this.driver.sendCommand('Fetch.enable', { patterns: [{requestStage: 'Request'}, {requestStage: 'Response'}], }); await this.driver.on('Fetch.requestPaused', this._onRequestPaused); } + async disableRequestInterception() { + if (!this._enabled) return; + + this._enabled = false; + await this.driver.off('Fetch.requestPaused', this._onRequestPaused); + await this.driver.sendCommand('Fetch.disable'); + this._onRequestPausedHandlers.clear(); + } + /** * @param {string} url * @param {(event: LH.Crdp.Fetch.RequestPausedEvent) => void} handler */ - async setOnRequestPausedHandler(url, handler) { + async _setOnRequestPausedHandler(url, handler) { this._onRequestPausedHandlers.set(url, handler); } @@ -66,12 +79,6 @@ class Fetcher { } } - async disableRequestInterception() { - await this.driver.off('Fetch.requestPaused', this._onRequestPaused); - await this.driver.sendCommand('Fetch.disable'); - this._onRequestPausedHandlers.clear(); - } - /** * Requires that `driver.enableRequestInterception` has been called. * @@ -82,13 +89,13 @@ class Fetcher { * @return {Promise} */ async fetchResource(url, {timeout = 500}) { - if (!this.driver.isDomainEnabled('Fetch')) { + if (!this._enabled) { throw new Error('Must call `enableRequestInterception` before using fetchResource'); } /** @type {Promise} */ const requestInterceptionPromise = new Promise((resolve, reject) => { - this.setOnRequestPausedHandler(url, async (event) => { + this._setOnRequestPausedHandler(url, async (event) => { const {requestId, responseStatusCode} = event; // The first requestPaused event is for the request stage. Continue it. From 32847c9dc5d2fb5c39a8b1c6850f71f3ebfc37e2 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Mar 2020 11:21:42 -0800 Subject: [PATCH 15/20] smoke, cleanup --- .../test/fixtures/source-map/script.js.map | 8 ++++ .../source-map/source-map-tester.html | 30 +++++++++++++ .../smokehouse/test-definitions/core-tests.js | 4 ++ .../source-maps/expectations.js | 42 +++++++++++++++++++ .../source-maps/source-maps-config.js | 26 ++++++++++++ lighthouse-core/gather/gather-runner.js | 10 +++++ .../gather/gatherers/source-maps.js | 3 +- 7 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 lighthouse-cli/test/fixtures/source-map/script.js.map create mode 100644 lighthouse-cli/test/fixtures/source-map/source-map-tester.html create mode 100644 lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js create mode 100644 lighthouse-cli/test/smokehouse/test-definitions/source-maps/source-maps-config.js diff --git a/lighthouse-cli/test/fixtures/source-map/script.js.map b/lighthouse-cli/test/fixtures/source-map/script.js.map new file mode 100644 index 000000000000..8bab5ab197de --- /dev/null +++ b/lighthouse-cli/test/fixtures/source-map/script.js.map @@ -0,0 +1,8 @@ +{ + "version": 3, + "file": "out.js", + "sourceRoot": "", + "sources": ["foo.js", "bar.js"], + "names": ["src", "maps", "are", "fun"], + "mappings": "AAgBC,SAAQ,CAAEA" +} diff --git a/lighthouse-cli/test/fixtures/source-map/source-map-tester.html b/lighthouse-cli/test/fixtures/source-map/source-map-tester.html new file mode 100644 index 000000000000..515607c646eb --- /dev/null +++ b/lighthouse-cli/test/fixtures/source-map/source-map-tester.html @@ -0,0 +1,30 @@ + + + + + + + + + Source maps tester + + + + + + + + map time map time! map time map time! 🎉 + + + \ No newline at end of file diff --git a/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js b/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js index d667dd605b64..4d91a03339ec 100644 --- a/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js +++ b/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js @@ -74,6 +74,10 @@ const smokeTests = [{ id: 'legacy-javascript', expectations: require('./legacy-javascript/expectations.js'), config: require('./legacy-javascript/legacy-javascript-config.js'), +},{ + id: 'source-maps', + expectations: require('./source-maps/expectations.js'), + config: require('./source-maps/source-maps-config.js'), }]; module.exports = smokeTests; diff --git a/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js b/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js new file mode 100644 index 000000000000..be66adbe8018 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/test-definitions/source-maps/expectations.js @@ -0,0 +1,42 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const fs = require('fs'); + +const mapPath = require.resolve('../../../fixtures/source-map/script.js.map'); +const mapJson = fs.readFileSync(mapPath, 'utf-8'); +const map = JSON.parse(mapJson); + +/** + * @type {Array} + * Expected Lighthouse audit values for seo tests + */ +const expectations = [ + { + artifacts: { + SourceMaps: [ + { + scriptUrl: 'http://localhost:10200/source-map/source-map-tester.html', + sourceMapUrl: 'http://localhost:10200/source-map/script.js.map', + map, + }, + { + scriptUrl: 'http://localhost:10200/source-map/source-map-tester.html', + sourceMapUrl: 'http://localhost:10503/source-map/script.js.map', + map, + }, + ], + }, + lhr: { + requestedUrl: 'http://localhost:10200/source-map/source-map-tester.html', + finalUrl: 'http://localhost:10200/source-map/source-map-tester.html', + audits: {}, + }, + }, +]; + +module.exports = expectations; diff --git a/lighthouse-cli/test/smokehouse/test-definitions/source-maps/source-maps-config.js b/lighthouse-cli/test/smokehouse/test-definitions/source-maps/source-maps-config.js new file mode 100644 index 000000000000..12ede17b3903 --- /dev/null +++ b/lighthouse-cli/test/smokehouse/test-definitions/source-maps/source-maps-config.js @@ -0,0 +1,26 @@ +/** + * @license Copyright 2019 Google Inc. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +/** + * Config file for running source map smokehouse. + */ + +// source-maps currently isn't in the default config yet, so we make a new one with it. +// Also, no audits use source-maps yet, and at least one is required for a successful run, +// so `viewport` and its required gatherer `meta-elements` is used. + +/** @type {LH.Config.Json} */ +module.exports = { + passes: [{ + passName: 'defaultPass', + gatherers: [ + 'source-maps', + 'meta-elements', + ], + }], + audits: ['viewport'], +}; diff --git a/lighthouse-core/gather/gather-runner.js b/lighthouse-core/gather/gather-runner.js index 25535144c0b7..7e07e4b52097 100644 --- a/lighthouse-core/gather/gather-runner.js +++ b/lighthouse-core/gather/gather-runner.js @@ -131,6 +131,11 @@ class GatherRunner { const resetStorage = !options.settings.disableStorageReset; if (resetStorage) await driver.clearDataForOrigin(options.requestedUrl); + // Disable fetcher, in case a gatherer enabled it. + // This cleanup should be removed once the only usage of + // fetcher (fetching arbitrary URLs) is replaced by new protocol support. + await driver.fetcher.disableRequestInterception(); + await driver.disconnect(); } catch (err) { // Ignore disconnecting error if browser was already closed. @@ -644,6 +649,11 @@ class GatherRunner { await GatherRunner.populateBaseArtifacts(passContext); isFirstPass = false; } + + // Disable fetcher, in case a gatherer enabled it. + // This cleanup should be removed once the only usage of + // fetcher (fetching arbitrary URLs) is replaced by new protocol support. + await driver.fetcher.disableRequestInterception(); } await GatherRunner.disposeDriver(driver, options); diff --git a/lighthouse-core/gather/gatherers/source-maps.js b/lighthouse-core/gather/gatherers/source-maps.js index 3e43af60499f..e0d0cef2a3b3 100644 --- a/lighthouse-core/gather/gatherers/source-maps.js +++ b/lighthouse-core/gather/gatherers/source-maps.js @@ -134,8 +134,7 @@ class SourceMaps extends Gatherer { await driver.fetcher.enableRequestInterception(); const eventProcessPromises = this._scriptParsedEvents .map((event) => this._retrieveMapFromScriptParsedEvent(driver, event)); - return Promise.all(eventProcessPromises) - .finally(() => driver.fetcher.disableRequestInterception()); + return Promise.all(eventProcessPromises); } } From 8ed46c283689a28de83478ae37a4438dbf6b7a29 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Mar 2020 14:18:13 -0800 Subject: [PATCH 16/20] comment --- lighthouse-core/gather/fetcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index cb51bdbad188..d2fb61f60af3 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -37,7 +37,7 @@ class Fetcher { * 3) if multiple commands to continue the same request are sent, protocol errors occur. * * So instead we have one global `Fetch.enable` / `Fetch.requestPaused` pair, and allow specific - * urls to be intercepted via `driver._setOnRequestPausedHandler`. + * urls to be intercepted via `fetcher._setOnRequestPausedHandler`. */ async enableRequestInterception() { if (this._enabled) return; From 1ebdc3398b14cd37ccd9bff18e9c42b7fac91a15 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Mar 2020 14:20:39 -0800 Subject: [PATCH 17/20] update --- .../test/smokehouse/test-definitions/core-tests.js | 2 +- lighthouse-core/gather/fetcher.js | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js b/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js index 4d91a03339ec..84d92541b860 100644 --- a/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js +++ b/lighthouse-cli/test/smokehouse/test-definitions/core-tests.js @@ -74,7 +74,7 @@ const smokeTests = [{ id: 'legacy-javascript', expectations: require('./legacy-javascript/expectations.js'), config: require('./legacy-javascript/legacy-javascript-config.js'), -},{ +}, { id: 'source-maps', expectations: require('./source-maps/expectations.js'), config: require('./source-maps/source-maps-config.js'), diff --git a/lighthouse-core/gather/fetcher.js b/lighthouse-core/gather/fetcher.js index d2fb61f60af3..061a0f51b5e9 100644 --- a/lighthouse-core/gather/fetcher.js +++ b/lighthouse-core/gather/fetcher.js @@ -101,11 +101,11 @@ class Fetcher { // The first requestPaused event is for the request stage. Continue it. if (!responseStatusCode) { // Remove cookies so we aren't buying stuff on Amazon. - const headers = []; - for (const [key, value] of Object.entries(event.request.headers)) { - if (key === 'Cookie') return; - headers.push({name: key, value}); - } + const headers = Object.entries(event.request.headers) + .filter(([name]) => name !== 'Cookie') + .map(([name, value]) => { + return {name, value}; + }); this.driver.sendCommand('Fetch.continueRequest', { requestId, From 0d7c09aadce8104b85e8a3e96c810a3fe52e163a Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Mar 2020 17:03:22 -0800 Subject: [PATCH 18/20] fix --- lighthouse-core/gather/driver.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lighthouse-core/gather/driver.js b/lighthouse-core/gather/driver.js index a5a373e45fb6..65a2235de67f 100644 --- a/lighthouse-core/gather/driver.js +++ b/lighthouse-core/gather/driver.js @@ -92,6 +92,7 @@ class Driver { */ this._nextProtocolTimeout = DEFAULT_PROTOCOL_TIMEOUT; + /** @type {Fetcher} */ this.fetcher = new Fetcher(this); } From ea35a533e42b49821e3e8f856d4bef41af5da810 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Wed, 4 Mar 2020 17:08:07 -0800 Subject: [PATCH 19/20] fix --- lighthouse-core/test/gather/fake-driver.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lighthouse-core/test/gather/fake-driver.js b/lighthouse-core/test/gather/fake-driver.js index 43fb64de61cb..19556a2ad538 100644 --- a/lighthouse-core/test/gather/fake-driver.js +++ b/lighthouse-core/test/gather/fake-driver.js @@ -12,6 +12,11 @@ function makeFakeDriver({protocolGetVersionResponse}) { let scrollPosition = {x: 0, y: 0}; return { + get fetcher() { + return { + disableRequestInterception: () => Promise.resolve(), + }; + }, getBrowserVersion() { return Promise.resolve(Object.assign({}, protocolGetVersionResponse, {milestone: 71})); }, From adc0adf86ece016c50d7dd08ec365991e50a2348 Mon Sep 17 00:00:00 2001 From: Connor Clark Date: Thu, 5 Mar 2020 13:30:50 -0800 Subject: [PATCH 20/20] comment --- lighthouse-core/gather/gather-runner.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lighthouse-core/gather/gather-runner.js b/lighthouse-core/gather/gather-runner.js index 7e07e4b52097..21bf8dcc9c63 100644 --- a/lighthouse-core/gather/gather-runner.js +++ b/lighthouse-core/gather/gather-runner.js @@ -650,7 +650,8 @@ class GatherRunner { isFirstPass = false; } - // Disable fetcher, in case a gatherer enabled it. + // Disable fetcher for every pass, in case a gatherer enabled it. + // Noop if fetcher was never enabled. // This cleanup should be removed once the only usage of // fetcher (fetching arbitrary URLs) is replaced by new protocol support. await driver.fetcher.disableRequestInterception();