[Unified Recorder] Call proxy-tool through dev-tool

common/tools/dev-tool/package.json

@@ -40,6 +40,7 @@
   "private": true,
   "prettier": "../eslint-plugin-azure-sdk/prettier.json",
   "dependencies": {
+    "concurrently": "^6.3.0",
     "chalk": "~4.1.1",
     "dotenv": "^8.2.0",
     "fs-extra": "^8.1.0",
@@ -55,6 +56,7 @@
     "@rollup/plugin-json": "^4.0.0",
     "@rollup/plugin-multi-entry": "^3.0.0",
     "@rollup/plugin-node-resolve": "^8.0.0",
+    "@types/concurrently": "^6.3.0",
     "@types/chai": "^4.1.6",
     "@types/chai-as-promised": "^7.1.0",
     "@types/fs-extra": "^8.0.0",

common/tools/dev-tool/src/commands/index.ts

@@ -12,7 +12,9 @@ const log = createPrinter("dev-tool");
 export const baseCommands = {
   about: () => import("./about"),
   package: () => import("./package"),
-  samples: () => import("./samples")
+  samples: () => import("./samples"),
+  "test-proxy": () => import("./test-proxy"),
+  "test:node": () => import("./testNode")
 } as const;
 
 /**

common/tools/dev-tool/src/commands/test-proxy/index.ts

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license
+
+import { subCommand, makeCommandInfo } from "../../framework/command";
+
+export const commandInfo = makeCommandInfo(
+  "test-proxy",
+  "runs the proxy-tool with the `docker run ...` command"
+);
+
+export default subCommand(commandInfo, {
+  start: () => import("./start")
+});

common/tools/dev-tool/src/commands/test-proxy/start.ts

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { leafCommand, makeCommandInfo } from "../../framework/command";
+import { config } from "dotenv";
+import { isProxyToolActive, startProxyTool } from "../../util/testProxyUtils";
+config();
+
+export const commandInfo = makeCommandInfo(
+  "test-proxy",
+  "runs the proxy-tool with the `docker run ...` command",
+  {}
+);
+
+export default leafCommand(commandInfo, async (_options) => {
+  const mode = process.env.TEST_MODE;
+  if (mode === "live") {
+    return true; // No need to start the proxy tool in the live mode
+  } else {
+    try {
+      await isProxyToolActive();
+      // No need to run a new one if it is already active
+      // Especially, CI uses this path
+      console.log(
+        `Proxy tool seems to be active, not attempting to start the test proxy at http://localhost:5000 & https://localhost:5001.\n`
+      );
+      return true;
+    } catch (error) {
+      if ((error as { code: string }).code === "ECONNREFUSED") {
+        // Proxy tool is not active, attempt to start the proxy tool now
+        await startProxyTool(mode);
+        return true;
+      } else {
+        throw error;
+      }
+    }
+  }
+});

common/tools/dev-tool/src/commands/testNode.ts

@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license
+
+import { leafCommand, makeCommandInfo } from "../framework/command";
+import concurrently from "concurrently";
+import { config } from "dotenv";
+import { isProxyToolActive } from "../util/testProxyUtils";
+config();
+
+export const commandInfo = makeCommandInfo(
+  "test:node",
+  "runs the node tests using mocha with the default and the provided options; starts the proxy-tool in record and playback modes",
+  {
+    "mocha-options": {
+      kind: "string",
+      description: "Mocha options along with the test files(glob pattern) as expected by mocha",
+      default: ""
+    }
+  }
+);
+
+export default leafCommand(commandInfo, async (_) => {
+  if (process.argv[3] !== "--mocha-options" && !process.argv[4]) {
+    throw new Error(
+      "unexpected command provided; expected = `dev-tool test:node --mocha-options '<options>'`"
+    );
+  }
+
+  const testProxyStart = "dev-tool test-proxy start";
+  const mochaCMDWithDefaults =
+    "nyc mocha -r esm --require ts-node/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace";
+  const mochaCommand = `${mochaCMDWithDefaults} ${process.argv[4]}`;
+
+  let runOnlyMochaCommand = false; // Boolean to figure out if we need to run just the mocha command or the test-proxy too
+
+  const mode = process.env.TEST_MODE;
+  if (mode === "live") {
+    runOnlyMochaCommand = true; // No need to start the proxy tool in the live mode
+  } else {
+    try {
+      await isProxyToolActive();
+      // No need to run a new one if it is already active
+      // Especially, CI uses this path
+      console.log(
+        `Proxy tool seems to be active, not attempting to start the test proxy at http://localhost:5000 & https://localhost:5001.\n`
+      );
+      runOnlyMochaCommand = true;
+    } catch (error) {
+      if ((error as { code: string }).code === "ECONNREFUSED") {
+        // Proxy tool is not active, attempt to start the proxy tool now
+        runOnlyMochaCommand = false;
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  const mochaCommandObj: concurrently.CommandObj = {
+    command: mochaCommand,
+    name: "node-tests"
+  };
+
+  if (runOnlyMochaCommand) {
+    await concurrently([mochaCommandObj]);
+  } else {
+    await concurrently([{ command: testProxyStart, name: "test-proxy" }, mochaCommandObj], {
+      killOthers: ["failure", "success"],
+      successCondition: "first"
+    });
+  }
+  return true;
+});

common/tools/dev-tool/src/util/testProxyUtils.ts

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { spawn } from "child_process";
+import fs from "fs";
+import path from "path";
+import { IncomingMessage, request, RequestOptions } from "http";
+
+export async function startProxyTool(mode: string | undefined) {
+  const outFileName = "test-proxy-output.log";
+  const out = fs.openSync(`./${outFileName}`, "a");
+  const err = fs.openSync(`./${outFileName}`, "a");
+
+  console.log(`===TEST_MODE="${mode}"===`);
+  console.log(
+    `Attempting to start test proxy at http://localhost:5000 & https://localhost:5001.\n`
+  );
+
+  const command = await getDockerRunCommand();
+
+  console.log(`Check the output file "${outFileName}" for test-proxy logs.`);
+
+  spawn(command, [], {
+    shell: true,
+    stdio: ["ignore", out, err]
+  });
+
+  // If you want it to be detached, add the following option and call unref()
+  // // detached: true,
+  // subprocess.unref();
+}
+
+function getRootLocation() {
+  let currentPath = process.cwd(); // Gives the current working directory
+  if (fs.existsSync(path.join(currentPath, "package.json"))) {
+    // <root>/sdk/service/project/package.json
+    const expectedRootPath = path.join(currentPath, "..", "..", ".."); // <root>/
+    if (
+      fs.existsSync(path.join(expectedRootPath, "sdk/")) && // <root>/sdk
+      fs.existsSync(path.join(expectedRootPath, "rush.json")) // <root>/rush.json
+    ) {
+      // reached root path
+      return expectedRootPath;
+    } else {
+      throw new Error("rootPath could not be calculated properly from process.cwd()");
+    }
+  } else {
+    throw new Error(`Expected 'package.json' to be found at ${currentPath}`);
+  }
+}
+
+async function getDockerRunCommand() {
+  const repoRoot = getRootLocation(); // /workspaces/azure-sdk-for-js/
+  const testProxyRecordingsLocation = "/etc/testproxy";
+  const allowLocalhostAccess = "--add-host host.docker.internal:host-gateway";
+  const imageToLoad = `azsdkengsys.azurecr.io/engsys/testproxy-lin:${await getImageTag()}`;
+  return `docker run -v ${repoRoot}:${testProxyRecordingsLocation} -p 5001:5001 -p 5000:5000 ${allowLocalhostAccess} ${imageToLoad}`;
+}
+
+export async function isProxyToolActive() {
+  await makeRequest("http://localhost:5000/info/available", {});
+  console.log(`Proxy tool seems to be active at http://localhost:5000\n`);
+}
+
+async function makeRequest(uri: string, requestOptions: RequestOptions): Promise<IncomingMessage> {
+  return new Promise<IncomingMessage>((resolve, reject) => {
+    let req = request(uri, requestOptions, resolve);
+    req.once("error", reject);
+    req.end();
+  });
+}
+
+async function getImageTag() {
+  // Grab the tag from the `/eng/common/testproxy/docker-start-proxy.ps1` file [..is used to run the proxy-tool in the CI]
+  //
+  // $SELECTED_IMAGE_TAG = "1147815";
+  // (Bot regularly updates the tag in the file above.)
+  try {
+    const contentInPWSHScript = await fs.promises.readFile(
+      `${path.join(getRootLocation(), "eng/common/testproxy/docker-start-proxy.ps1")}`,
+      "utf-8"
+    );
+    const tag = contentInPWSHScript.match(/\$SELECTED_IMAGE_TAG \= \"(.*)\"/)![1];
+    console.log(`Image tag obtained from the powershell script => ${tag}\n`);
+    return tag;
+  } catch (_) {
+    console.log(
+      `Unable to get the image tag from the powershell script, trying "latest" tag instead\n`
+    );
+    return "latest";
+  }
+}

eng/tools/rush-runner.js

@@ -111,7 +111,7 @@ const getServicePackages = (baseDir, serviceDirs) => {
     const packageJsons = getPackageJsons(searchDir);
     for (const filePath of packageJsons) {
       const contents = JSON.parse(fs.readFileSync(filePath, "utf8"));
-      if (contents["sdk-type"] === "client" || contents["sdk-type"] === "mgmt" || contents["sdk-type"] === "perf-test") {
+      if (contents["sdk-type"] === "client" || contents["sdk-type"] === "mgmt" || contents["sdk-type"] === "perf-test" || contents["sdk-type"] === "utility") {
         packageNames.push(contents.name);
         packageDirs.push(path.dirname(filePath));
       }

sdk/test-utils/perf/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@azure/test-utils-perf",
   "version": "1.0.0",
-  "sdk-type": "client",
+  "sdk-type": "utility",
   "description": "Performance and stress test framework for the Azure SDK for JavaScript and TypeScript",
   "main": "dist-esm/src/index.js",
   "module": "dist-esm/src/index.js",

sdk/test-utils/recorder-new/package.json

@@ -22,21 +22,22 @@
     "extract-api": "echo skipped",
     "format": "prettier --write --config ../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"",
     "integration-test:browser": "echo skipped",
-    "integration-test:node": "echo skipped",
+    "integration-test:node": "concurrently \"npm run tests:server\" \"npm run test:node-with-proxy\" --kill-others --success first",
+    "test:node-with-proxy": "dev-tool test:node --mocha-options \"--timeout 1200000 'test/*.spec.ts'\"",
     "integration-test": "npm run integration-test:node && npm run integration-test:browser",
     "tests:server": "cross-env TS_NODE_COMPILER_OPTIONS=\"{\\\"module\\\": \\\"commonjs\\\"}\" ts-node test/utils/server.ts",
     "temp-integration-test:browser": "concurrently \"npm run tests:server\" \"karma start --single-run\" --kill-others --success first",
-    "temp-integration-test:node": "concurrently \"npm run tests:server\" \"nyc mocha -r esm --require ts-node/register --reporter ../../../common/tools/mocha-multi-reporter.js --timeout 5000000 --full-trace 'test/*.spec.ts'\" --kill-others --success first",
     "lint:fix": "eslint package.json src test --ext .ts --fix --fix-type [problem,suggestion]",
     "lint": "eslint package.json src test --ext .ts -f html -o recorder-lintReport.html || exit 0",
     "pack": "npm pack 2>&1",
     "unit-test:browser": "echo skipped",
     "unit-test:node": "echo skipped",
     "unit-test": "npm run unit-test:node && npm run unit-test:browser",
     "test:browser": "npm run clean && npm run build && npm run temp-integration-test:browser",
-    "test:node": "npm run clean && npm run build:test && npm run temp-integration-test:node",
+    "test:node": "npm run clean && npm run build:test && npm run integration-test:node",
     "test": "npm run clean && npm run build:test && npm run unit-test",
-    "docs": "echo Skipped."
+    "docs": "echo Skipped.",
+    "dev-tool": "dev-tool"
   },
   "files": [
     "dist/",

sdk/test-utils/recorder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@azure-tools/test-recorder",
   "version": "1.0.2",
-  "sdk-type": "client",
+  "sdk-type": "utility",
   "description": "This library provides interfaces and helper methods to provide recording and playback capabilities for the tests in Azure JS/TS SDKs",
   "main": "dist/index.js",
   "module": "dist-esm/src/index.js",

sdk/test-utils/test-utils/package.json

@@ -3,6 +3,7 @@
   "version": "1.0.0",
   "description": "Test utilities library for the Azure SDK for JavaScript and TypeScript",
   "main": "dist/index.js",
+  "sdk-type": "utility",
   "module": "dist-esm/src/index.js",
   "browser": {
     "./dist-esm/src/global.js": "./dist-esm/src/global.browser.js"

sdk/test-utils/testing-recorder-new/README.md

@@ -12,3 +12,4 @@ You will need to create a local `.env` file under the same directory as this rea
       "STORAGE_CONNECTION_STRING",
       "STORAGE_SAS_URL",
       "TABLES_SAS_CONNECTION_STRING"
+```

sdk/test-utils/testing-recorder-new/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@azure-tools/testing-recorder-new",
   "version": "1.0.0",
-  "sdk-type": "test",
+  "sdk-type": "client",
   "description": "This library uses the interfaces and helper methods from @azure-tools/test-recorder-new in the tests to show an example",
   "main": "dist/index.js",
   "module": "dist-esm/src/index.js",
@@ -18,7 +18,8 @@
     "extract-api": "echo skipped",
     "format": "prettier --write --config ../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"",
     "integration-test:browser": "karma start --single-run",
-    "integration-test:node": "mocha -r esm --require ts-node/register --reporter ../../../common/tools/mocha-multi-reporter.js --timeout 1200000 --full-trace \"test/*.spec.ts\"",
+    "integration-test:node": "dev-tool test:node --mocha-options \"--timeout 1200000 'test/*.spec.ts'\"",
+    "backup-integration-test:node": "mocha -r esm --require ts-node/register --reporter ../../../common/tools/mocha-multi-reporter.js --timeout 1200000 --full-trace \"test/*.spec.ts\"",
     "integration-test": "npm run integration-test:node && npm run integration-test:browser",
     "lint:fix": "eslint package.json src test --ext .ts --fix --fix-type [problem,suggestion]",
     "lint": "eslint package.json src test --ext .ts -f html -o recorder-lintReport.html || exit 0",