FTR support mocha dry run (elastic#125915)

This PR adds support for the mocha CLI flag `--dry-run` in our functional test execution.

package.json

@@ -637,7 +637,7 @@
     "@types/mime-types": "^2.1.0",
     "@types/minimatch": "^2.0.29",
     "@types/minimist": "^1.2.1",
-    "@types/mocha": "^8.2.0",
+    "@types/mocha": "^9.1.0",
     "@types/mock-fs": "^4.13.1",
     "@types/moment-timezone": "^0.5.12",
     "@types/mustache": "^0.8.31",
@@ -775,7 +775,7 @@
     "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jest": "^24.5.0",
     "eslint-plugin-jsx-a11y": "^6.4.1",
-    "eslint-plugin-mocha": "^9.0.0",
+    "eslint-plugin-mocha": "^10.0.3",
     "eslint-plugin-no-unsanitized": "^3.1.5",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
@@ -832,10 +832,10 @@
     "micromatch": "3.1.10",
     "minimist": "^1.2.5",
     "mkdirp": "0.5.1",
-    "mocha": "^8.2.1",
-    "mocha-junit-reporter": "^2.0.0",
-    "mochawesome": "^6.2.1",
-    "mochawesome-merge": "^4.2.0",
+    "mocha": "^9.1.0",
+    "mocha-junit-reporter": "^2.0.2",
+    "mochawesome": "^7.0.1",
+    "mochawesome-merge": "^4.2.1",
     "mock-fs": "^5.1.2",
     "mock-http-server": "1.3.0",
     "ms-chromium-edge-driver": "^0.4.3",

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -46,6 +46,7 @@ export function runFtrCli() {
         {
           mochaOpts: {
             bail: flags.bail,
+            dryRun: flags['dry-run'],
             grep: flags.grep || undefined,
             invert: flags.invert,
           },
@@ -148,6 +149,7 @@ export function runFtrCli() {
           'u',
           'throttle',
           'headless',
+          'dry-run',
         ],
         default: {
           config: 'test/functional/config.js',
@@ -174,6 +176,7 @@ export function runFtrCli() {
           --kibana-install-dir  directory where the Kibana install being tested resides
           --throttle         enable network throttling in Chrome browser
           --headless         run browser in headless mode
+          --dry-run          report tests without executing them
         `,
       },
     }

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -6,7 +6,9 @@
  * Side Public License, v 1.
  */
 
+import Path from 'path';
 import { ToolingLog } from '@kbn/dev-utils';
+import { REPO_ROOT } from '@kbn/utils';
 
 import { Suite, Test } from './fake_mocha_types';
 import {
@@ -74,7 +76,28 @@ export class FunctionalTestRunner {
         return (await providers.invokeProviderFn(customTestRunner)) || 0;
       }
 
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
+      let reporter;
+      let reporterOptions;
+      if (config.get('mochaOpts.dryRun')) {
+        // override default reporter for dryRun results
+        const targetFile = Path.resolve(REPO_ROOT, 'target/functional-tests/dryRunOutput.json');
+        reporter = 'json';
+        reporterOptions = {
+          output: targetFile,
+        };
+        this.log.info(`Dry run results will be stored in ${targetFile}`);
+      }
+
+      const mocha = await setupMocha(
+        this.lifecycle,
+        this.log,
+        config,
+        providers,
+        this.esVersion,
+        reporter,
+        reporterOptions
+      );
+
       await this.lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -123,6 +123,7 @@ export const schema = Joi.object()
     mochaOpts: Joi.object()
       .keys({
         bail: Joi.boolean().default(false),
+        dryRun: Joi.boolean().default(false),
         grep: Joi.string(),
         invert: Joi.boolean().default(false),
         slow: Joi.number().default(30000),

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -25,11 +25,21 @@ import { validateCiGroupTags } from './validate_ci_group_tags';
  *  @param  {EsVersion} esVersion
  *  @return {Promise<Mocha>}
  */
-export async function setupMocha(lifecycle, log, config, providers, esVersion) {
+export async function setupMocha(
+  lifecycle,
+  log,
+  config,
+  providers,
+  esVersion,
+  reporter,
+  reporterOptions
+) {
   // configure mocha
   const mocha = new Mocha({
     ...config.get('mochaOpts'),
-    reporter: await providers.loadExternalService('mocha reporter', MochaReporterProvider),
+    reporter:
+      reporter || (await providers.loadExternalService('mocha reporter', MochaReporterProvider)),
+    reporterOptions,
   });
 
   // global beforeEach hook in root suite triggers before all others

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -65,6 +65,7 @@ const options = {
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
   silent: { desc: 'Log nothing.' },
+  'dry-run': { desc: 'Report tests without executing them.' },
 };
 
 export function displayHelp() {
@@ -136,6 +137,11 @@ export function processOptions(userOptions, defaultConfigPaths) {
   userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded'];
   delete userOptions['assert-none-excluded'];
 
+  if (userOptions['dry-run']) {
+    userOptions.dryRun = userOptions['dry-run'];
+    delete userOptions['dry-run'];
+  }
+
   function createLogger() {
     return new ToolingLog({
       level: pickLevelFromFlags(userOptions),

packages/kbn-test/src/functional_tests/lib/run_ftr.ts

@@ -27,6 +27,7 @@ export interface CreateFtrOptions {
   };
   updateSnapshots?: boolean;
   esVersion: EsVersion;
+  dryRun?: boolean;
 }
 
 export interface CreateFtrParams {
@@ -45,6 +46,7 @@ async function createFtr({
     suiteTags,
     updateSnapshots,
     esVersion,
+    dryRun,
   },
 }: CreateFtrParams) {
   const config = await readConfigFile(log, esVersion, configPath);
@@ -58,6 +60,7 @@ async function createFtr({
         mochaOpts: {
           bail: !!bail,
           grep,
+          dryRun: !!dryRun,
         },
         kbnTestServer: {
           installDir,