tests: smoke request count assertion

lighthouse-cli/test/fixtures/static-server.js

@@ -28,6 +28,8 @@ class Server {
     this._server = http.createServer(this._requestHandler.bind(this));
     /** @type {(data: string) => string=} */
     this._dataTransformer = undefined;
+    /** @type {string[]} */
+    this._requestUrls = [];
   }
 
   getPort() {
@@ -62,8 +64,30 @@ class Server {
     this._dataTransformer = fn;
   }
 
+  /**
+   * @return {string[]}
+   */
+  takeRequestUrls() {
+    const requestUrls = this._requestUrls;
+    this._requestUrls = [];
+    return requestUrls;
+  }
+
+  /**
+   * @param {http.IncomingMessage} request
+   */
+  _updateRequestUrls(request) {
+    if (['/favicon.ico', '/robots.txt'].includes(request.url)) return;
+    this._requestUrls.push(request.url);
+  }
+
+  /**
+   * @param {http.IncomingMessage} request
+   * @param {http.ServerResponse} response
+   */
   _requestHandler(request, response) {
     const requestUrl = parseURL(request.url);
+    this._updateRequestUrls(request);
     const filePath = requestUrl.pathname;
     const queryString = requestUrl.search && parseQueryString(requestUrl.search.slice(1));
     let absoluteFilePath = path.join(this.baseDir, filePath);

lighthouse-cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -132,7 +132,11 @@ async function begin() {
   const invertMatch = argv.invertMatch;
   const testDefns = getDefinitionsToRun(allTestDefns, requestedTestIds, {invertMatch});
 
-  const options = {jobs, retries, isDebug: argv.debug, lighthouseRunner};
+  const takeNetworkRequestUrls = () => {
+    return server.takeRequestUrls();
+  };
+
+  const options = {jobs, retries, isDebug: argv.debug, lighthouseRunner, takeNetworkRequestUrls};
 
   let isPassing;
   try {

lighthouse-cli/test/smokehouse/report-assert.js

@@ -157,13 +157,13 @@ function makeComparison(name, actualResult, expectedResult) {
  * @param {LocalConsole} localConsole
  * @param {LH.Result} lhr
  * @param {Smokehouse.ExpectedRunnerResult} expected
+ * @param {boolean|undefined} isSync
  */
-function pruneExpectations(localConsole, lhr, expected) {
+function pruneExpectations(localConsole, lhr, expected, isSync) {
   const userAgent = lhr.environment.hostUserAgent;
   const userAgentMatch = /Chrome\/(\d+)/.exec(userAgent); // Chrome/85.0.4174.0
   if (!userAgentMatch) throw new Error('Could not get chrome version.');
   const actualChromeVersion = Number(userAgentMatch[1]);
-
   /**
    * @param {*} obj
    */
@@ -199,14 +199,25 @@ function pruneExpectations(localConsole, lhr, expected) {
   }
 
   const cloned = cloneDeep(expected);
+
+  if (!isSync && expected.networkRequests) {
+    const msg = 'Network requests should only be asserted on tests run synchronously';
+    if (process.env.CI) {
+      throw new Error(msg);
+    } else {
+      localConsole.log(`${msg}, pruning expectation: ${JSON.stringify(expected.networkRequests)}`);
+      delete cloned.networkRequests;
+    }
+  }
+
   pruneNewerChromeExpectations(cloned);
   return cloned;
 }
 
 /**
  * Collate results into comparisons of actual and expected scores on each audit/artifact.
  * @param {LocalConsole} localConsole
- * @param {{lhr: LH.Result, artifacts: LH.Artifacts}} actual
+ * @param {{lhr: LH.Result, artifacts: LH.Artifacts, networkRequests: string[]}} actual
  * @param {Smokehouse.ExpectedRunnerResult} expected
  * @return {Comparison[]}
  */
@@ -254,6 +265,16 @@ function collateResults(localConsole, actual, expected) {
     return makeComparison(auditName + ' audit', actualResult, expectedResult);
   });
 
+  /** @type {Comparison[]} */
+  const requestCountAssertion = [];
+  if (expected.networkRequests) {
+    requestCountAssertion.push(makeComparison(
+      'Requests',
+      actual.networkRequests,
+      expected.networkRequests
+    ));
+  }
+
   return [
     {
       name: 'final url',
@@ -263,6 +284,7 @@ function collateResults(localConsole, actual, expected) {
     },
     runtimeErrorAssertion,
     runWarningsAssertion,
+    ...requestCountAssertion,
     ...artifactAssertions,
     ...auditAssertions,
   ];
@@ -334,15 +356,15 @@ function assertLogString(count) {
 /**
  * Log all the comparisons between actual and expected test results, then print
  * summary. Returns count of passed and failed tests.
- * @param {{lhr: LH.Result, artifacts: LH.Artifacts}} actual
+ * @param {{lhr: LH.Result, artifacts: LH.Artifacts, networkRequests: string[]}} actual
  * @param {Smokehouse.ExpectedRunnerResult} expected
- * @param {{isDebug?: boolean}=} reportOptions
+ * @param {{isDebug?: boolean, isSync?: boolean}=} reportOptions
  * @return {{passed: number, failed: number, log: string}}
  */
 function report(actual, expected, reportOptions = {}) {
   const localConsole = new LocalConsole();
 
-  expected = pruneExpectations(localConsole, actual.lhr, expected);
+  expected = pruneExpectations(localConsole, actual.lhr, expected, reportOptions.isSync);
   const comparisons = collateResults(localConsole, actual, expected);
 
   let correctCount = 0;

lighthouse-cli/test/smokehouse/smokehouse.js

@@ -44,6 +44,7 @@ async function runSmokehouse(smokeTestDefns, smokehouseOptions = {}) {
     jobs = DEFAULT_CONCURRENT_RUNS,
     retries = DEFAULT_RETRIES,
     lighthouseRunner = cliLighthouseRunner,
+    takeNetworkRequestUrls = () => [],
   } = smokehouseOptions;
   assertPositiveInteger('jobs', jobs);
   assertNonNegativeInteger('retries', retries);
@@ -54,7 +55,7 @@ async function runSmokehouse(smokeTestDefns, smokehouseOptions = {}) {
   for (const testDefn of smokeTestDefns) {
     // If defn is set to `runSerially`, we'll run its tests in succession, not parallel.
     const concurrency = testDefn.runSerially ? 1 : jobs;
-    const options = {concurrency, lighthouseRunner, retries, isDebug};
+    const options = {concurrency, lighthouseRunner, retries, isDebug, takeNetworkRequestUrls};
     const result = runSmokeTestDefn(concurrentMapper, testDefn, options);
     smokePromises.push(result);
   }
@@ -110,21 +111,25 @@ function assertNonNegativeInteger(loggableName, value) {
  * once all are finished.
  * @param {ConcurrentMapper} concurrentMapper
  * @param {Smokehouse.TestDfn} smokeTestDefn
- * @param {{concurrency: number, retries: number, lighthouseRunner: Smokehouse.LighthouseRunner, isDebug?: boolean}} defnOptions
+ * @param {{concurrency: number, retries: number, lighthouseRunner: Smokehouse.LighthouseRunner, isDebug?: boolean, takeNetworkRequestUrls: () => string[]}} defnOptions
  * @return {Promise<SmokehouseResult>}
  */
 async function runSmokeTestDefn(concurrentMapper, smokeTestDefn, defnOptions) {
   const {id, config: configJson, expectations} = smokeTestDefn;
-  const {concurrency, lighthouseRunner, retries, isDebug} = defnOptions;
+  const {concurrency, lighthouseRunner, retries, isDebug, takeNetworkRequestUrls} = defnOptions;
 
-  const individualTests = expectations.map(expectation => ({
-    requestedUrl: expectation.lhr.requestedUrl,
-    configJson,
-    expectation,
-    lighthouseRunner,
-    retries,
-    isDebug,
-  }));
+  const individualTests = expectations.map(expectation => {
+    return {
+      requestedUrl: expectation.lhr.requestedUrl,
+      configJson,
+      expectation,
+      lighthouseRunner,
+      retries,
+      isDebug,
+      isSync: concurrency === 1,
+      takeNetworkRequestUrls,
+    };
+  });
 
   // Loop sequentially over expectations, comparing against Lighthouse run, and
   // reporting result.
@@ -171,14 +176,23 @@ function purpleify(str) {
 /**
  * Run Lighthouse in the selected runner. Returns `log`` for logging once
  * all tests in a defn are complete.
- * @param {{requestedUrl: string, configJson?: LH.Config.Json, expectation: Smokehouse.ExpectedRunnerResult, lighthouseRunner: Smokehouse.LighthouseRunner, retries: number, isDebug?: boolean}} testOptions
+ * @param {{requestedUrl: string, configJson?: LH.Config.Json, expectation: Smokehouse.ExpectedRunnerResult, lighthouseRunner: Smokehouse.LighthouseRunner, retries: number, isDebug?: boolean, isSync?: boolean, takeNetworkRequestUrls: () => string[]}} testOptions
  * @return {Promise<{passed: number, failed: number, log: string}>}
  */
 async function runSmokeTest(testOptions) {
   // Use a buffered LocalConsole to keep logged output so it's not interleaved
   // with other currently running tests.
   const localConsole = new LocalConsole();
-  const {requestedUrl, configJson, expectation, lighthouseRunner, retries, isDebug} = testOptions;
+  const {
+    requestedUrl,
+    configJson,
+    expectation,
+    lighthouseRunner,
+    retries,
+    isDebug,
+    isSync,
+    takeNetworkRequestUrls,
+  } = testOptions;
 
   // Rerun test until there's a passing result or retries are exhausted to prevent flakes.
   let result;
@@ -192,14 +206,17 @@ async function runSmokeTest(testOptions) {
 
     // Run Lighthouse.
     try {
-      result = await lighthouseRunner(requestedUrl, configJson, {isDebug});
+      result = {
+        ...await lighthouseRunner(requestedUrl, configJson, {isDebug}),
+        ...{networkRequests: takeNetworkRequestUrls()},
+      };
     } catch (e) {
       logChildProcessError(localConsole, e);
       continue; // Retry, if possible.
     }
 
     // Assert result.
-    report = getAssertionReport(result, expectation, {isDebug});
+    report = getAssertionReport(result, expectation, {isDebug, isSync});
     if (report.failed) {
       localConsole.log(`${report.failed} assertion(s) failed.`);
       continue; // Retry, if possible.

lighthouse-cli/test/smokehouse/test-definitions/perf/expectations.js

@@ -11,6 +11,9 @@
  */
 module.exports = [
   {
+    networkRequests: {
+      length: 9,
+    },
     lhr: {
       requestedUrl: 'http://localhost:10200/preload.html',
       finalUrl: 'http://localhost:10200/preload.html',
@@ -62,6 +65,9 @@ module.exports = [
     },
   },
   {
+    networkRequests: {
+      length: 8,
+    },
     lhr: {
       requestedUrl: 'http://localhost:10200/perf/perf-budgets/load-things.html',
       finalUrl: 'http://localhost:10200/perf/perf-budgets/load-things.html',
@@ -140,6 +146,9 @@ module.exports = [
     },
   },
   {
+    networkRequests: {
+      length: 5,
+    },
     lhr: {
       requestedUrl: 'http://localhost:10200/perf/fonts.html',
       finalUrl: 'http://localhost:10200/perf/fonts.html',
@@ -168,6 +177,9 @@ module.exports = [
     },
   },
   {
+    networkRequests: {
+      length: 3,
+    },
     artifacts: {
       TraceElements: [
         {
@@ -291,6 +303,9 @@ module.exports = [
     },
   },
   {
+    networkRequests: {
+      length: 2,
+    },
     lhr: {
       requestedUrl: 'http://localhost:10200/perf/frame-metrics.html',
       finalUrl: 'http://localhost:10200/perf/frame-metrics.html',

types/smokehouse.d.ts

@@ -19,6 +19,7 @@
   export type ExpectedRunnerResult = {
     lhr: ExpectedLHR,
     artifacts?: Partial<Record<keyof LH.Artifacts, any>>
+    networkRequests?: {length: number};
   }
 
   export interface TestDfn {
@@ -41,6 +42,8 @@
     retries?: number;
     /** A function that runs Lighthouse with the given options. Defaults to running Lighthouse via the CLI. */
     lighthouseRunner?: LighthouseRunner;
+    /** A function that gets a list of URLs requested to the server since the last fetch. */
+    takeNetworkRequestUrls?: () => string[];
   }
 
   export interface SmokehouseLibOptions extends SmokehouseOptions {