Skip to content

Commit

Permalink
feat(ng-dev): create workflow performance testing tooling (#2418)
Browse files Browse the repository at this point in the history
Creates a piece of tooling within ng-dev that allows for a set of commands to be run that emulate an expected
workflow within a repository. This is then measured for the time it takes to run these commands so they can be
checked or tracked over time.

PR Close #2418
  • Loading branch information
josephperrott committed Nov 6, 2024
1 parent 7898454 commit db24afe
Show file tree
Hide file tree
Showing 17 changed files with 478 additions and 22 deletions.
47 changes: 43 additions & 4 deletions .github/local-actions/branch-manager/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -56692,7 +56692,7 @@ var supportsColor2 = {
var supports_color_default2 = supportsColor2;

//
import { spawn as _spawn, spawnSync as _spawnSync } from "child_process";
import { spawn as _spawn, spawnSync as _spawnSync, exec as _exec } from "child_process";
var ChildProcess = class {
static spawnInteractive(command, args, options = {}) {
return new Promise((resolve, reject) => {
Expand All @@ -56706,7 +56706,7 @@ var ChildProcess = class {
return new Promise((resolve, reject) => {
const commandText = `${command} ${args.join(" ")}`;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const childProcess = _spawn(command, args, { ...options, env: env3, shell: true, stdio: "pipe" });
let logOutput = "";
Expand Down Expand Up @@ -56747,7 +56747,7 @@ ${logOutput}`);
}
static spawnSync(command, args, options = {}) {
const commandText = `${command} ${args.join(" ")}`;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const { status: exitCode, signal, stdout, stderr } = _spawnSync(command, args, { ...options, env: env3, encoding: "utf8", shell: true, stdio: "pipe" });
const status = statusFromExitCodeAndSignal(exitCode, signal);
Expand All @@ -56756,11 +56756,50 @@ ${logOutput}`);
}
throw new Error(stderr);
}
static exec(command, options = {}) {
return new Promise((resolve, reject) => {
var _a2, _b;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${command}`);
const childProcess = _exec(command, { ...options, env: env3 });
let logOutput = "";
let stdout = "";
let stderr = "";
(_a2 = childProcess.stderr) == null ? void 0 : _a2.on("data", (message) => {
stderr += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
(_b = childProcess.stdout) == null ? void 0 : _b.on("data", (message) => {
stdout += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
childProcess.on("close", (exitCode, signal) => {
const exitDescription = exitCode !== null ? `exit code "${exitCode}"` : `signal "${signal}"`;
const printFn = outputMode === "on-error" ? Log.error : Log.debug;
const status = statusFromExitCodeAndSignal(exitCode, signal);
printFn(`Command "${command}" completed with ${exitDescription}.`);
printFn(`Process output:
${logOutput}`);
if (status === 0 || options.suppressErrorOnFailingExitCode) {
resolve({ stdout, stderr, status });
} else {
reject(outputMode === "silent" ? logOutput : void 0);
}
});
});
}
};
function statusFromExitCodeAndSignal(exitCode, signal) {
return exitCode ?? signal ?? -1;
}
function getEnvironmentForNonInteractiveSpawn(userProvidedEnv) {
function getEnvironmentForNonInteractiveCommand(userProvidedEnv) {
const forceColorValue = supports_color_default2.stdout !== false ? supports_color_default2.stdout.level.toString() : void 0;
return { FORCE_COLOR: forceColorValue, ...userProvidedEnv ?? process.env };
}
Expand Down
47 changes: 43 additions & 4 deletions .github/local-actions/changelog/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -56565,7 +56565,7 @@ var supportsColor2 = {
var supports_color_default2 = supportsColor2;

//
import { spawn as _spawn, spawnSync as _spawnSync } from "child_process";
import { spawn as _spawn, spawnSync as _spawnSync, exec as _exec } from "child_process";
var ChildProcess = class {
static spawnInteractive(command, args, options = {}) {
return new Promise((resolve, reject) => {
Expand All @@ -56579,7 +56579,7 @@ var ChildProcess = class {
return new Promise((resolve, reject) => {
const commandText = `${command} ${args.join(" ")}`;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const childProcess = _spawn(command, args, { ...options, env: env3, shell: true, stdio: "pipe" });
let logOutput = "";
Expand Down Expand Up @@ -56620,7 +56620,7 @@ ${logOutput}`);
}
static spawnSync(command, args, options = {}) {
const commandText = `${command} ${args.join(" ")}`;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const { status: exitCode, signal, stdout, stderr } = _spawnSync(command, args, { ...options, env: env3, encoding: "utf8", shell: true, stdio: "pipe" });
const status = statusFromExitCodeAndSignal(exitCode, signal);
Expand All @@ -56629,11 +56629,50 @@ ${logOutput}`);
}
throw new Error(stderr);
}
static exec(command, options = {}) {
return new Promise((resolve, reject) => {
var _a2, _b;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${command}`);
const childProcess = _exec(command, { ...options, env: env3 });
let logOutput = "";
let stdout = "";
let stderr = "";
(_a2 = childProcess.stderr) == null ? void 0 : _a2.on("data", (message) => {
stderr += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
(_b = childProcess.stdout) == null ? void 0 : _b.on("data", (message) => {
stdout += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
childProcess.on("close", (exitCode, signal) => {
const exitDescription = exitCode !== null ? `exit code "${exitCode}"` : `signal "${signal}"`;
const printFn = outputMode === "on-error" ? Log.error : Log.debug;
const status = statusFromExitCodeAndSignal(exitCode, signal);
printFn(`Command "${command}" completed with ${exitDescription}.`);
printFn(`Process output:
${logOutput}`);
if (status === 0 || options.suppressErrorOnFailingExitCode) {
resolve({ stdout, stderr, status });
} else {
reject(outputMode === "silent" ? logOutput : void 0);
}
});
});
}
};
function statusFromExitCodeAndSignal(exitCode, signal) {
return exitCode ?? signal ?? -1;
}
function getEnvironmentForNonInteractiveSpawn(userProvidedEnv) {
function getEnvironmentForNonInteractiveCommand(userProvidedEnv) {
const forceColorValue = supports_color_default2.stdout !== false ? supports_color_default2.stdout.level.toString() : void 0;
return { FORCE_COLOR: forceColorValue, ...userProvidedEnv ?? process.env };
}
Expand Down
16 changes: 16 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,19 @@ jobs:
- uses: ./github-actions/bazel/setup
- run: yarn install --immutable
- run: yarn bazel test --sandbox_writable_path="$HOME/Library/Application Support" --test_tag_filters=macos --build_tests_only -- //...

workflow-perf:
timeout-minutes: 30
runs-on: ubuntu-latest
steps:
# Because the checkout and setup node action is contained in the dev-infra repo, we must
# checkout the repo to be able to run the action we have created. Other repos will skip
# this step.
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./github-actions/npm/checkout-and-setup-node
- uses: ./github-actions/bazel/setup
- run: yarn install --immutable
- run: yarn ng-dev perf workflows --json
# Always run this step to ensure that the job always is successful
- if: ${{ always() }}
run: exit 0
17 changes: 17 additions & 0 deletions .ng-dev/dx-perf-workflows.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
workflows:
- name: Rerun a test
prepare:
- bazel clean;
- bazel build //ng-dev/utils/test;
workflow:
- bazel test //ng-dev/utils/test;
- git apply .ng-dev/perf-tests/test-rerun.diff;
- bazel test //ng-dev/utils/test;
cleanup:
- git apply -R .ng-dev/perf-tests/test-rerun.diff;

- name: Build Everything
prepare:
- bazel clean;
workflow:
- bazel build //...;
17 changes: 17 additions & 0 deletions .ng-dev/perf-tests/test-rerun.diff
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
diff --git a/ng-dev/utils/test/g3.spec.ts b/ng-dev/utils/test/g3.spec.ts
index a82c1b7a..8e0b24f8 100644
--- a/ng-dev/utils/test/g3.spec.ts
+++ b/ng-dev/utils/test/g3.spec.ts
@@ -29,9 +29,9 @@ describe('G3Stats', () => {
});

function setupFakeSyncConfig(config: GoogleSyncConfig): string {
- const configFileName = 'sync-test-conf.json';
- fs.writeFileSync(path.join(git.baseDir, configFileName), JSON.stringify(config));
- return configFileName;
+ const somethingelse = 'sync-test-conf.json';
+ fs.writeFileSync(path.join(git.baseDir, somethingelse), JSON.stringify(config));
+ return somethingelse;
}

describe('gathering stats', () => {
47 changes: 43 additions & 4 deletions github-actions/create-pr-for-changes/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -41501,7 +41501,7 @@ var supportsColor2 = {
var supports_color_default2 = supportsColor2;

//
import { spawn as _spawn, spawnSync as _spawnSync } from "child_process";
import { spawn as _spawn, spawnSync as _spawnSync, exec as _exec } from "child_process";
var ChildProcess = class {
static spawnInteractive(command, args, options = {}) {
return new Promise((resolve, reject) => {
Expand All @@ -41515,7 +41515,7 @@ var ChildProcess = class {
return new Promise((resolve, reject) => {
const commandText = `${command} ${args.join(" ")}`;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const childProcess = _spawn(command, args, { ...options, env: env3, shell: true, stdio: "pipe" });
let logOutput = "";
Expand Down Expand Up @@ -41556,7 +41556,7 @@ ${logOutput}`);
}
static spawnSync(command, args, options = {}) {
const commandText = `${command} ${args.join(" ")}`;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const { status: exitCode, signal, stdout, stderr } = _spawnSync(command, args, { ...options, env: env3, encoding: "utf8", shell: true, stdio: "pipe" });
const status = statusFromExitCodeAndSignal(exitCode, signal);
Expand All @@ -41565,11 +41565,50 @@ ${logOutput}`);
}
throw new Error(stderr);
}
static exec(command, options = {}) {
return new Promise((resolve, reject) => {
var _a, _b;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${command}`);
const childProcess = _exec(command, { ...options, env: env3 });
let logOutput = "";
let stdout = "";
let stderr = "";
(_a = childProcess.stderr) == null ? void 0 : _a.on("data", (message) => {
stderr += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
(_b = childProcess.stdout) == null ? void 0 : _b.on("data", (message) => {
stdout += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
childProcess.on("close", (exitCode, signal) => {
const exitDescription = exitCode !== null ? `exit code "${exitCode}"` : `signal "${signal}"`;
const printFn = outputMode === "on-error" ? Log.error : Log.debug;
const status = statusFromExitCodeAndSignal(exitCode, signal);
printFn(`Command "${command}" completed with ${exitDescription}.`);
printFn(`Process output:
${logOutput}`);
if (status === 0 || options.suppressErrorOnFailingExitCode) {
resolve({ stdout, stderr, status });
} else {
reject(outputMode === "silent" ? logOutput : void 0);
}
});
});
}
};
function statusFromExitCodeAndSignal(exitCode, signal) {
return exitCode ?? signal ?? -1;
}
function getEnvironmentForNonInteractiveSpawn(userProvidedEnv) {
function getEnvironmentForNonInteractiveCommand(userProvidedEnv) {
const forceColorValue = supports_color_default2.stdout !== false ? supports_color_default2.stdout.level.toString() : void 0;
return { FORCE_COLOR: forceColorValue, ...userProvidedEnv ?? process.env };
}
Expand Down
47 changes: 43 additions & 4 deletions github-actions/slash-commands/main.js
Original file line number Diff line number Diff line change
Expand Up @@ -53425,7 +53425,7 @@ var supportsColor2 = {
var supports_color_default2 = supportsColor2;

//
import { spawn as _spawn, spawnSync as _spawnSync } from "child_process";
import { spawn as _spawn, spawnSync as _spawnSync, exec as _exec } from "child_process";
var ChildProcess = class {
static spawnInteractive(command, args, options = {}) {
return new Promise((resolve, reject) => {
Expand All @@ -53439,7 +53439,7 @@ var ChildProcess = class {
return new Promise((resolve, reject) => {
const commandText = `${command} ${args.join(" ")}`;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const childProcess = _spawn(command, args, { ...options, env: env3, shell: true, stdio: "pipe" });
let logOutput = "";
Expand Down Expand Up @@ -53480,7 +53480,7 @@ ${logOutput}`);
}
static spawnSync(command, args, options = {}) {
const commandText = `${command} ${args.join(" ")}`;
const env3 = getEnvironmentForNonInteractiveSpawn(options.env);
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${commandText}`);
const { status: exitCode, signal, stdout, stderr } = _spawnSync(command, args, { ...options, env: env3, encoding: "utf8", shell: true, stdio: "pipe" });
const status = statusFromExitCodeAndSignal(exitCode, signal);
Expand All @@ -53489,11 +53489,50 @@ ${logOutput}`);
}
throw new Error(stderr);
}
static exec(command, options = {}) {
return new Promise((resolve, reject) => {
var _a2, _b;
const outputMode = options.mode;
const env3 = getEnvironmentForNonInteractiveCommand(options.env);
Log.debug(`Executing command: ${command}`);
const childProcess = _exec(command, { ...options, env: env3 });
let logOutput = "";
let stdout = "";
let stderr = "";
(_a2 = childProcess.stderr) == null ? void 0 : _a2.on("data", (message) => {
stderr += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
(_b = childProcess.stdout) == null ? void 0 : _b.on("data", (message) => {
stdout += message;
logOutput += message;
if (outputMode === void 0 || outputMode === "enabled") {
process.stderr.write(message);
}
});
childProcess.on("close", (exitCode, signal) => {
const exitDescription = exitCode !== null ? `exit code "${exitCode}"` : `signal "${signal}"`;
const printFn = outputMode === "on-error" ? Log.error : Log.debug;
const status = statusFromExitCodeAndSignal(exitCode, signal);
printFn(`Command "${command}" completed with ${exitDescription}.`);
printFn(`Process output:
${logOutput}`);
if (status === 0 || options.suppressErrorOnFailingExitCode) {
resolve({ stdout, stderr, status });
} else {
reject(outputMode === "silent" ? logOutput : void 0);
}
});
});
}
};
function statusFromExitCodeAndSignal(exitCode, signal) {
return exitCode ?? signal ?? -1;
}
function getEnvironmentForNonInteractiveSpawn(userProvidedEnv) {
function getEnvironmentForNonInteractiveCommand(userProvidedEnv) {
const forceColorValue = supports_color_default2.stdout !== false ? supports_color_default2.stdout.level.toString() : void 0;
return { FORCE_COLOR: forceColorValue, ...userProvidedEnv ?? process.env };
}
Expand Down
1 change: 1 addition & 0 deletions ng-dev/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ ts_library(
"//ng-dev/format",
"//ng-dev/misc",
"//ng-dev/ngbot",
"//ng-dev/perf",
"//ng-dev/pr",
"//ng-dev/pr/common/labels",
"//ng-dev/pr/config",
Expand Down
2 changes: 2 additions & 0 deletions ng-dev/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {buildReleaseParser} from './release/cli.js';
import {tsCircularDependenciesBuilder} from './ts-circular-dependencies/index.js';
import {captureLogOutputForCommand} from './utils/logging.js';
import {buildAuthParser} from './auth/cli.js';
import {buildPerfParser} from './perf/cli.js';
import {Argv} from 'yargs';

runParserWithCompletedFunctions((yargs: Argv) => {
Expand All @@ -38,6 +39,7 @@ runParserWithCompletedFunctions((yargs: Argv) => {
.command('caretaker <command>', '', buildCaretakerParser)
.command('misc <command>', '', buildMiscParser)
.command('ngbot <command>', false, buildNgbotParser)
.command('perf <command>', '', buildPerfParser)
.wrap(120)
.strict();
});
Loading

0 comments on commit db24afe

Please sign in to comment.