tests: run most smoke tests on devtools #13456
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -47,6 +47,7 @@ yarn-error.log | |
|
|
||
| /chrome-linux/ | ||
| /chrome-win32/ | ||
| /chrome-mac/ | ||
| /chrome.zip | ||
|
|
||
| *__pycache__ | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,98 @@ | ||||||
| /** | ||||||
| * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved. | ||||||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||||||
| * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. | ||||||
| */ | ||||||
| 'use strict'; | ||||||
|
|
||||||
| /** | ||||||
| * @fileoverview A runner that launches Chrome and executes Lighthouse via DevTools. | ||||||
| */ | ||||||
|
|
||||||
| import fs from 'fs'; | ||||||
| import os from 'os'; | ||||||
| import {spawn} from 'child_process'; | ||||||
|
|
||||||
| import {LH_ROOT} from '../../../../root.js'; | ||||||
|
|
||||||
| const devtoolsDir = | ||||||
| process.env.DEVTOOLS_PATH || `${LH_ROOT}/.tmp/chromium-web-tests/devtools/devtools-frontend`; | ||||||
|
|
||||||
| /** | ||||||
| * @param {string} command | ||||||
| * @param {string[]} args | ||||||
| */ | ||||||
| async function spawnAndLog(command, args) { | ||||||
| let log = ''; | ||||||
|
|
||||||
| /** @type {Promise<void>} */ | ||||||
| const promise = new Promise((resolve, reject) => { | ||||||
| const spawnHandle = spawn(command, args); | ||||||
| spawnHandle.on('close', code => { | ||||||
| if (code === 0) resolve(); | ||||||
| else reject(new Error(`Command exited with code ${code}`)); | ||||||
| }); | ||||||
| spawnHandle.on('error', reject); | ||||||
| spawnHandle.stdout.on('data', data => { | ||||||
| console.log(data.toString()); | ||||||
|
There was a problem hiding this comment. are these lines intended to stay or for debugging? There was a problem hiding this comment. Yes, otherwise the cases where things hang or error aren't printed (the runner interface only allows returning successful logs). and it's nicer to see the output stream in. There was a problem hiding this comment. just checking that it's intentional. The tradeoff is that concurrent runs will be streaming in stuff all interleaved with each other and hard to tell the source of each line, but that can still be helpful, especially when things are new and need to check that it's actually running, etc |
||||||
| log += `STDOUT: ${data.toString()}`; | ||||||
| }); | ||||||
| spawnHandle.stderr.on('data', data => { | ||||||
| console.log(data.toString()); | ||||||
| log += `STDERR: ${data.toString()}`; | ||||||
| }); | ||||||
| }); | ||||||
| await promise; | ||||||
|
|
||||||
| return log; | ||||||
| } | ||||||
|
|
||||||
| /** @type {Promise<void>} */ | ||||||
| let beforeAllPromise; | ||||||
|
There was a problem hiding this comment. maybe
Suggested change
or something like that? |
||||||
| async function beforeAll() { | ||||||
|
There was a problem hiding this comment. can you add a jsdoc like, "Build the lighthouse DevTools bundle and fetch and build the DevTools frontend using it" or whatever? |
||||||
| if (process.env.CI) return; | ||||||
|
|
||||||
| process.env.DEVTOOLS_PATH = devtoolsDir; | ||||||
| await spawnAndLog('bash', ['lighthouse-core/test/chromium-web-tests/download-devtools.sh']); | ||||||
| await spawnAndLog('bash', ['lighthouse-core/test/chromium-web-tests/roll-devtools.sh']); | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Launch Chrome and do a full Lighthouse run via DevTools. | ||||||
|
There was a problem hiding this comment. I'm having trouble coming up with a non-awkward way to say this, but seems worth clarifying it's the current/local Lighthouse devtools bundle run in the latest DevTools frontend |
||||||
| * @param {string} url | ||||||
| * @param {LH.Config.Json=} configJson | ||||||
| * @param {{isDebug?: boolean}=} testRunnerOptions | ||||||
| * @return {Promise<{lhr: LH.Result, artifacts: LH.Artifacts, log: string}>} | ||||||
| */ | ||||||
| async function runLighthouse(url, configJson, testRunnerOptions = {}) { | ||||||
| if (!beforeAllPromise) beforeAllPromise = beforeAll(); | ||||||
| await beforeAllPromise; | ||||||
|
|
||||||
| const outputDir = fs.mkdtempSync(os.tmpdir() + '/lh-smoke-cdt-runner-'); | ||||||
|
There was a problem hiding this comment. Does this create multiple temp directories if the smokes are running concurrently? There was a problem hiding this comment. yeah, |
||||||
| const args = [ | ||||||
| 'run-devtools', | ||||||
| url, | ||||||
| `--custom-devtools-frontend=file://${devtoolsDir}/out/Default/gen/front_end`, | ||||||
| '--output-dir', outputDir, | ||||||
|
There was a problem hiding this comment. We could use the custom DT frontend flag to test how PR changes affect DT. There was a problem hiding this comment. Ah, I keep making this mistake, thinking that is the default behavior. It is for open-devtools! I want to followup with a change that does that. Will add the arg here for now. |
||||||
| ]; | ||||||
| if (configJson) { | ||||||
| args.push('--config', JSON.stringify(configJson)); | ||||||
| } | ||||||
|
|
||||||
| const log = await spawnAndLog('yarn', args); | ||||||
| const lhr = JSON.parse(fs.readFileSync(`${outputDir}/lhr-0.json`, 'utf-8')); | ||||||
| const artifacts = JSON.parse(fs.readFileSync(`${outputDir}/artifacts-0.json`, 'utf-8')); | ||||||
|
|
||||||
| if (testRunnerOptions.isDebug) { | ||||||
| console.log(`${url} results saved at ${outputDir}`); | ||||||
| } else { | ||||||
| fs.rmSync(outputDir, {recursive: true, force: true}); | ||||||
| } | ||||||
|
|
||||||
| return {lhr, artifacts, log}; | ||||||
| } | ||||||
|
|
||||||
| export { | ||||||
| beforeAll, | ||||||
| runLighthouse, | ||||||
| }; | ||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Better in a new job?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How to avoid redoing all the work for building devtools?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah it's not pretty. We could have dependent jobs (or cache the build, but dependent jobs seems workable)? I'm not sure of the best approach, but if the devtools CI job is normally going to take 30+ minutes we'll definitely want to split it up.
edit: if an already-built devtools was available, sharding would work as well