[8.0] FTR support mocha dry run (#125915) (#126237)

* FTR support mocha dry run (#125915)

This PR adds support for the mocha CLI flag `--dry-run` in our functional test execution.

(cherry picked from commit b7db60a)

# Conflicts:
#	yarn.lock

* Rebuild @kbn/pm

* Rebuild @kbn/pm part 2

Co-authored-by: Kibana Machine <[email protected]>

package.json

@@ -587,7 +587,7 @@
     "@types/mime-types": "^2.1.0",
     "@types/minimatch": "^2.0.29",
     "@types/minimist": "^1.2.1",
-    "@types/mocha": "^8.2.0",
+    "@types/mocha": "^9.1.0",
     "@types/mock-fs": "^4.13.1",
     "@types/moment-timezone": "^0.5.12",
     "@types/mustache": "^0.8.31",
@@ -725,7 +725,7 @@
     "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-jest": "^24.5.0",
     "eslint-plugin-jsx-a11y": "^6.4.1",
-    "eslint-plugin-mocha": "^9.0.0",
+    "eslint-plugin-mocha": "^10.0.3",
     "eslint-plugin-no-unsanitized": "^3.1.5",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prefer-object-spread": "^1.2.1",
@@ -782,10 +782,10 @@
     "micromatch": "3.1.10",
     "minimist": "^1.2.5",
     "mkdirp": "0.5.1",
-    "mocha": "^8.2.1",
-    "mocha-junit-reporter": "^2.0.0",
-    "mochawesome": "^6.2.1",
-    "mochawesome-merge": "^4.2.0",
+    "mocha": "^9.1.0",
+    "mocha-junit-reporter": "^2.0.2",
+    "mochawesome": "^7.0.1",
+    "mochawesome-merge": "^4.2.1",
     "mock-fs": "^5.1.2",
     "mock-http-server": "1.3.0",
     "ms-chromium-edge-driver": "^0.4.3",

packages/kbn-pm/dist/index.js

@@ -18605,7 +18605,6 @@ function pauseStreams (streams, options) {
 
 module.exports = glob
 
-var fs = __webpack_require__(132)
 var rp = __webpack_require__(245)
 var minimatch = __webpack_require__(247)
 var Minimatch = minimatch.Minimatch
@@ -18616,8 +18615,6 @@ var assert = __webpack_require__(162)
 var isAbsolute = __webpack_require__(253)
 var globSync = __webpack_require__(254)
 var common = __webpack_require__(255)
-var alphasort = common.alphasort
-var alphasorti = common.alphasorti
 var setopts = common.setopts
 var ownProp = common.ownProp
 var inflight = __webpack_require__(256)
@@ -19068,7 +19065,7 @@ Glob.prototype._readdirInGlobStar = function (abs, cb) {
   var lstatcb = inflight(lstatkey, lstatcb_)
 
   if (lstatcb)
-    fs.lstat(abs, lstatcb)
+    self.fs.lstat(abs, lstatcb)
 
   function lstatcb_ (er, lstat) {
     if (er && er.code === 'ENOENT')
@@ -19109,7 +19106,7 @@ Glob.prototype._readdir = function (abs, inGlobStar, cb) {
   }
 
   var self = this
-  fs.readdir(abs, readdirCb(this, abs, cb))
+  self.fs.readdir(abs, readdirCb(this, abs, cb))
 }
 
 function readdirCb (self, abs, cb) {
@@ -19313,13 +19310,13 @@ Glob.prototype._stat = function (f, cb) {
   var self = this
   var statcb = inflight('stat\0' + abs, lstatcb_)
   if (statcb)
-    fs.lstat(abs, statcb)
+    self.fs.lstat(abs, statcb)
 
   function lstatcb_ (er, lstat) {
     if (lstat && lstat.isSymbolicLink()) {
       // If it's a symlink, then treat it as the target, unless
       // the target does not exist, then treat it as a file.
-      return fs.stat(abs, function (er, stat) {
+      return self.fs.stat(abs, function (er, stat) {
         if (er)
           self._stat2(f, abs, null, lstat, cb)
         else
@@ -21039,7 +21036,6 @@ module.exports.win32 = win32;
 module.exports = globSync
 globSync.GlobSync = GlobSync
 
-var fs = __webpack_require__(132)
 var rp = __webpack_require__(245)
 var minimatch = __webpack_require__(247)
 var Minimatch = minimatch.Minimatch
@@ -21049,8 +21045,6 @@ var path = __webpack_require__(4)
 var assert = __webpack_require__(162)
 var isAbsolute = __webpack_require__(253)
 var common = __webpack_require__(255)
-var alphasort = common.alphasort
-var alphasorti = common.alphasorti
 var setopts = common.setopts
 var ownProp = common.ownProp
 var childrenIgnored = common.childrenIgnored
@@ -21286,7 +21280,7 @@ GlobSync.prototype._readdirInGlobStar = function (abs) {
   var lstat
   var stat
   try {
-    lstat = fs.lstatSync(abs)
+    lstat = this.fs.lstatSync(abs)
   } catch (er) {
     if (er.code === 'ENOENT') {
       // lstat failed, doesn't exist
@@ -21323,7 +21317,7 @@ GlobSync.prototype._readdir = function (abs, inGlobStar) {
   }
 
   try {
-    return this._readdirEntries(abs, fs.readdirSync(abs))
+    return this._readdirEntries(abs, this.fs.readdirSync(abs))
   } catch (er) {
     this._readdirError(abs, er)
     return null
@@ -21482,7 +21476,7 @@ GlobSync.prototype._stat = function (f) {
   if (!stat) {
     var lstat
     try {
-      lstat = fs.lstatSync(abs)
+      lstat = this.fs.lstatSync(abs)
     } catch (er) {
       if (er && (er.code === 'ENOENT' || er.code === 'ENOTDIR')) {
         this.statCache[abs] = false
@@ -21492,7 +21486,7 @@ GlobSync.prototype._stat = function (f) {
 
     if (lstat && lstat.isSymbolicLink()) {
       try {
-        stat = fs.statSync(abs)
+        stat = this.fs.statSync(abs)
       } catch (er) {
         stat = lstat
       }
@@ -21528,8 +21522,6 @@ GlobSync.prototype._makeAbs = function (f) {
 /* 255 */
 /***/ (function(module, exports, __webpack_require__) {
 
-exports.alphasort = alphasort
-exports.alphasorti = alphasorti
 exports.setopts = setopts
 exports.ownProp = ownProp
 exports.makeAbs = makeAbs
@@ -21542,17 +21534,14 @@ function ownProp (obj, field) {
   return Object.prototype.hasOwnProperty.call(obj, field)
 }
 
+var fs = __webpack_require__(132)
 var path = __webpack_require__(4)
 var minimatch = __webpack_require__(247)
 var isAbsolute = __webpack_require__(253)
 var Minimatch = minimatch.Minimatch
 
-function alphasorti (a, b) {
-  return a.toLowerCase().localeCompare(b.toLowerCase())
-}
-
 function alphasort (a, b) {
-  return a.localeCompare(b)
+  return a.localeCompare(b, 'en')
 }
 
 function setupIgnores (self, options) {
@@ -21611,6 +21600,7 @@ function setopts (self, pattern, options) {
   self.stat = !!options.stat
   self.noprocess = !!options.noprocess
   self.absolute = !!options.absolute
+  self.fs = options.fs || fs
 
   self.maxLength = options.maxLength || Infinity
   self.cache = options.cache || Object.create(null)
@@ -21680,7 +21670,7 @@ function finish (self) {
     all = Object.keys(all)
 
   if (!self.nosort)
-    all = all.sort(self.nocase ? alphasorti : alphasort)
+    all = all.sort(alphasort)
 
   // at *some* point we statted all of these
   if (self.mark) {

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -46,6 +46,7 @@ export function runFtrCli() {
         {
           mochaOpts: {
             bail: flags.bail,
+            dryRun: flags['dry-run'],
             grep: flags.grep || undefined,
             invert: flags.invert,
           },
@@ -148,6 +149,7 @@ export function runFtrCli() {
           'u',
           'throttle',
           'headless',
+          'dry-run',
         ],
         default: {
           config: 'test/functional/config.js',
@@ -174,6 +176,7 @@ export function runFtrCli() {
           --kibana-install-dir  directory where the Kibana install being tested resides
           --throttle         enable network throttling in Chrome browser
           --headless         run browser in headless mode
+          --dry-run          report tests without executing them
         `,
       },
     }

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -6,7 +6,9 @@
  * Side Public License, v 1.
  */
 
+import Path from 'path';
 import { ToolingLog } from '@kbn/dev-utils';
+import { REPO_ROOT } from '@kbn/utils';
 
 import { Suite, Test } from './fake_mocha_types';
 import {
@@ -74,7 +76,28 @@ export class FunctionalTestRunner {
         return (await providers.invokeProviderFn(customTestRunner)) || 0;
       }
 
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
+      let reporter;
+      let reporterOptions;
+      if (config.get('mochaOpts.dryRun')) {
+        // override default reporter for dryRun results
+        const targetFile = Path.resolve(REPO_ROOT, 'target/functional-tests/dryRunOutput.json');
+        reporter = 'json';
+        reporterOptions = {
+          output: targetFile,
+        };
+        this.log.info(`Dry run results will be stored in ${targetFile}`);
+      }
+
+      const mocha = await setupMocha(
+        this.lifecycle,
+        this.log,
+        config,
+        providers,
+        this.esVersion,
+        reporter,
+        reporterOptions
+      );
+
       await this.lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -123,6 +123,7 @@ export const schema = Joi.object()
     mochaOpts: Joi.object()
       .keys({
         bail: Joi.boolean().default(false),
+        dryRun: Joi.boolean().default(false),
         grep: Joi.string(),
         invert: Joi.boolean().default(false),
         slow: Joi.number().default(30000),

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -25,11 +25,21 @@ import { validateCiGroupTags } from './validate_ci_group_tags';
  *  @param  {EsVersion} esVersion
  *  @return {Promise<Mocha>}
  */
-export async function setupMocha(lifecycle, log, config, providers, esVersion) {
+export async function setupMocha(
+  lifecycle,
+  log,
+  config,
+  providers,
+  esVersion,
+  reporter,
+  reporterOptions
+) {
   // configure mocha
   const mocha = new Mocha({
     ...config.get('mochaOpts'),
-    reporter: await providers.loadExternalService('mocha reporter', MochaReporterProvider),
+    reporter:
+      reporter || (await providers.loadExternalService('mocha reporter', MochaReporterProvider)),
+    reporterOptions,
   });
 
   // global beforeEach hook in root suite triggers before all others

packages/kbn-test/src/functional_tests/cli/run_tests/args.js

@@ -65,6 +65,7 @@ const options = {
   debug: { desc: 'Run in debug mode.' },
   quiet: { desc: 'Only log errors.' },
   silent: { desc: 'Log nothing.' },
+  'dry-run': { desc: 'Report tests without executing them.' },
 };
 
 export function displayHelp() {
@@ -136,6 +137,11 @@ export function processOptions(userOptions, defaultConfigPaths) {
   userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded'];
   delete userOptions['assert-none-excluded'];
 
+  if (userOptions['dry-run']) {
+    userOptions.dryRun = userOptions['dry-run'];
+    delete userOptions['dry-run'];
+  }
+
   function createLogger() {
     return new ToolingLog({
       level: pickLevelFromFlags(userOptions),

packages/kbn-test/src/functional_tests/lib/run_ftr.ts

@@ -27,6 +27,7 @@ export interface CreateFtrOptions {
   };
   updateSnapshots?: boolean;
   esVersion: EsVersion;
+  dryRun?: boolean;
 }
 
 export interface CreateFtrParams {
@@ -45,6 +46,7 @@ async function createFtr({
     suiteTags,
     updateSnapshots,
     esVersion,
+    dryRun,
   },
 }: CreateFtrParams) {
   const config = await readConfigFile(log, esVersion, configPath);
@@ -58,6 +60,7 @@ async function createFtr({
         mochaOpts: {
           bail: !!bail,
           grep,
+          dryRun: !!dryRun,
         },
         kbnTestServer: {
           installDir,