From d2b4090217bfe6c7a5ae501f2a156da947aadb54 Mon Sep 17 00:00:00 2001 From: "Sander.Schutten@athora.nl" Date: Wed, 21 Jun 2023 14:00:15 +0200 Subject: [PATCH 1/4] Added console-error-checking --- README.md | 1 + src/setup-page.ts | 12 +++++++++++- src/test-storybook.ts | 4 ++++ src/util/getCliOptions.test.ts | 9 +++++++++ src/util/getCliOptions.ts | 2 ++ src/util/getParsedCliOptions.ts | 3 ++- 6 files changed, 29 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7f3a2608..6d0158ed 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ Usage: test-storybook [options] | `--junit` | Indicates that test information should be reported in a junit file.
`test-storybook --**junit**` | | `--ci` | Instead of the regular behavior of storing a new snapshot automatically, it will fail the test and require Jest to be run with `--updateSnapshot`.
`test-storybook --ci` | | `--shard [shardIndex/shardCount]` | Splits your test suite across different machines to run in CI.
`test-storybook --shard=1/3` | +| `--checkConsole` | Checks the browser console output for errors
`test-storybook --checkConsole` | ## Ejecting configuration diff --git a/src/setup-page.ts b/src/setup-page.ts index 64634e38..f3a3f1f5 100644 --- a/src/setup-page.ts +++ b/src/setup-page.ts @@ -51,6 +51,7 @@ const sanitizeURL = (url: string) => { export const setupPage = async (page: Page, browserContext: BrowserContext) => { const targetURL = process.env.TARGET_URL; + const checkConsole = process.env.TEST_CHECK_CONSOLE; const viewMode = process.env.VIEW_MODE || 'story'; const renderedEvent = viewMode === 'docs' ? 'docsRendered' : 'storyRendered'; @@ -306,10 +307,14 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => { // collect logs to show upon test error let logs = []; + let hasErrors = false; const spyOnConsole = (method, name) => { const originalFn = console[method]; return function () { + if (\`${checkConsole}\`==='true' && method==='error') { + hasErrors = true; + } const message = [...arguments].map(composeMessage).join(', '); const prefix = \`\${bold(name)}: \`; logs.push(prefix + message); @@ -332,7 +337,12 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => { }) return new Promise((resolve, reject) => { - channel.on('${renderedEvent}', () => resolve(document.getElementById('root'))); + channel.on('${renderedEvent}', () => { + if (hasErrors) { + return reject(new StorybookTestRunnerError(storyId, 'Browser console errors', logs)); + } + return resolve(document.getElementById('root')); + }); channel.on('storyUnchanged', () => resolve(document.getElementById('root'))); channel.on('storyErrored', ({ description }) => reject( new StorybookTestRunnerError(storyId, description, logs)) diff --git a/src/test-storybook.ts b/src/test-storybook.ts index b68566db..9fc8201b 100644 --- a/src/test-storybook.ts +++ b/src/test-storybook.ts @@ -297,6 +297,10 @@ const main = async () => { const { storiesPaths, lazyCompilation } = getStorybookMetadata(); process.env.STORYBOOK_STORIES_PATTERN = storiesPaths; + if (runnerOptions.checkConsole) { + process.env.TEST_CHECK_CONSOLE = 'true'; + } + if (lazyCompilation && isLocalStorybookIp) { log( `You're running Storybook with lazy compilation enabled, and will likely cause issues with the test runner locally. Consider disabling 'lazyCompilation' in ${runnerOptions.configDir}/main.js when running 'test-storybook' locally.` diff --git a/src/util/getCliOptions.test.ts b/src/util/getCliOptions.test.ts index ae4d2726..91ef708c 100644 --- a/src/util/getCliOptions.test.ts +++ b/src/util/getCliOptions.test.ts @@ -10,4 +10,13 @@ describe('getCliOptions', () => { const opts = getCliOptions(); expect(opts.runnerOptions).toMatchObject(customConfig); }); + + it('returns checkConsole option if passed', () => { + const customConfig = { checkConsole: true }; + jest + .spyOn(cliHelper, 'getParsedCliOptions') + .mockReturnValue({ options: customConfig, extraArgs: [] }); + const opts = getCliOptions(); + expect(opts.runnerOptions).toMatchObject(customConfig); + }); }); diff --git a/src/util/getCliOptions.ts b/src/util/getCliOptions.ts index 9e098a5b..47d0e181 100644 --- a/src/util/getCliOptions.ts +++ b/src/util/getCliOptions.ts @@ -14,6 +14,7 @@ export type CliOptions = { coverage?: boolean; junit?: boolean; browsers?: BrowserType | BrowserType[]; + checkConsole?: boolean; }; jestOptions: JestOptions; }; @@ -28,6 +29,7 @@ const STORYBOOK_RUNNER_COMMANDS: StorybookRunnerCommand[] = [ 'url', 'coverage', 'junit', + 'checkConsole', ]; function copyOption( diff --git a/src/util/getParsedCliOptions.ts b/src/util/getParsedCliOptions.ts index 67d200ad..a7b5f203 100644 --- a/src/util/getParsedCliOptions.ts +++ b/src/util/getParsedCliOptions.ts @@ -70,7 +70,8 @@ export const getParsedCliOptions = (): ParsedCliOptions => { .option( '--shard ', 'Splits your test suite across different machines to run in CI.' - ); + ) + .option('--checkConsole', 'Checks the browser console output for errors'); program.exitOverride(); From bc733b7b6913dc667dfd6cfec7e2100cbea74acb Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Fri, 4 Aug 2023 14:19:11 +0200 Subject: [PATCH 2/4] parse console errors when logging them in test results --- src/setup-page.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/setup-page.ts b/src/setup-page.ts index f3a3f1f5..ec0e5def 100644 --- a/src/setup-page.ts +++ b/src/setup-page.ts @@ -216,6 +216,9 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => { // end of fast-safe-stringify code function composeMessage(args) { + if (args instanceof Error) { + return \`\${args.name}: \${args.message}\\n\${args.stack}\`; + } if (typeof args === 'undefined') return "undefined"; if (typeof args === 'string') return args; return stringify(args, null, null, { depthLimit: 5, edgesLimit: 100 }); From c9d7f05fd8598b9fa1e1cb4ce05efd6ac99890b9 Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Fri, 4 Aug 2023 14:24:07 +0200 Subject: [PATCH 3/4] update cli flag name --- README.md | 2 +- src/setup-page.ts | 4 ++-- src/test-storybook.ts | 2 +- src/util/getCliOptions.test.ts | 4 ++-- src/util/getCliOptions.ts | 4 ++-- src/util/getParsedCliOptions.ts | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2dc3ba79..7716f97d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Usage: test-storybook [options] | `--junit` | Indicates that test information should be reported in a junit file.
`test-storybook --**junit**` | | `--ci` | Instead of the regular behavior of storing a new snapshot automatically, it will fail the test and require Jest to be run with `--updateSnapshot`.
`test-storybook --ci` | | `--shard [shardIndex/shardCount]` | Splits your test suite across different machines to run in CI.
`test-storybook --shard=1/3` | -| `--checkConsole` | Checks the browser console output for errors
`test-storybook --checkConsole` | +| `--failOnConsole` | Makes tests fail on browser console errors
`test-storybook --failOnConsole` | ## Ejecting configuration diff --git a/src/setup-page.ts b/src/setup-page.ts index ec0e5def..a4ac3acc 100644 --- a/src/setup-page.ts +++ b/src/setup-page.ts @@ -51,7 +51,7 @@ const sanitizeURL = (url: string) => { export const setupPage = async (page: Page, browserContext: BrowserContext) => { const targetURL = process.env.TARGET_URL; - const checkConsole = process.env.TEST_CHECK_CONSOLE; + const failOnConsole = process.env.TEST_CHECK_CONSOLE; const viewMode = process.env.VIEW_MODE || 'story'; const renderedEvent = viewMode === 'docs' ? 'docsRendered' : 'storyRendered'; @@ -315,7 +315,7 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => { const spyOnConsole = (method, name) => { const originalFn = console[method]; return function () { - if (\`${checkConsole}\`==='true' && method==='error') { + if (\`${failOnConsole}\`==='true' && method==='error') { hasErrors = true; } const message = [...arguments].map(composeMessage).join(', '); diff --git a/src/test-storybook.ts b/src/test-storybook.ts index 85cf5a90..17503f24 100644 --- a/src/test-storybook.ts +++ b/src/test-storybook.ts @@ -304,7 +304,7 @@ const main = async () => { const { storiesPaths, lazyCompilation } = getStorybookMetadata(); process.env.STORYBOOK_STORIES_PATTERN = storiesPaths; - if (runnerOptions.checkConsole) { + if (runnerOptions.failOnConsole) { process.env.TEST_CHECK_CONSOLE = 'true'; } diff --git a/src/util/getCliOptions.test.ts b/src/util/getCliOptions.test.ts index 539da184..e9e663a0 100644 --- a/src/util/getCliOptions.test.ts +++ b/src/util/getCliOptions.test.ts @@ -17,8 +17,8 @@ describe('getCliOptions', () => { expect(opts.runnerOptions).toMatchObject(customConfig); }); - it('returns checkConsole option if passed', () => { - const customConfig = { checkConsole: true }; + it('returns failOnConsole option if passed', () => { + const customConfig = { failOnConsole: true }; jest .spyOn(cliHelper, 'getParsedCliOptions') .mockReturnValue({ options: customConfig, extraArgs: [] }); diff --git a/src/util/getCliOptions.ts b/src/util/getCliOptions.ts index 7c74196c..2db332e1 100644 --- a/src/util/getCliOptions.ts +++ b/src/util/getCliOptions.ts @@ -13,7 +13,7 @@ export type CliOptions = { coverageDirectory?: string; junit?: boolean; browsers?: BrowserType | BrowserType[]; - checkConsole?: boolean; + failOnConsole?: boolean; }; jestOptions: JestOptions; }; @@ -29,7 +29,7 @@ const STORYBOOK_RUNNER_COMMANDS: StorybookRunnerCommand[] = [ 'coverage', 'coverageDirectory', 'junit', - 'checkConsole', + 'failOnConsole', ]; function copyOption( diff --git a/src/util/getParsedCliOptions.ts b/src/util/getParsedCliOptions.ts index 701cf7a8..d4df6774 100644 --- a/src/util/getParsedCliOptions.ts +++ b/src/util/getParsedCliOptions.ts @@ -76,7 +76,7 @@ export const getParsedCliOptions = (): ParsedCliOptions => { '--shard ', 'Splits your test suite across different machines to run in CI.' ) - .option('--checkConsole', 'Checks the browser console output for errors'); + .option('--failOnConsole', 'Makes tests fail on browser console errors'); program.exitOverride(); From 542f0b28ca2bb80e5335d7748492ee0cdb04a8ef Mon Sep 17 00:00:00 2001 From: Yann Braga Date: Fri, 4 Aug 2023 14:30:53 +0200 Subject: [PATCH 4/4] add story to test console error failures --- package.json | 2 +- stories/expected-failures/Failure.stories.tsx | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 9062cbb3..bc6eb5b9 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "build-storybook": "storybook build", "release": "yarn build && auto shipit", "test-storybook": "node dist/test-storybook", - "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json", + "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --failOnConsole --outputFile test-results.json", "test-storybook:no-cache": "yarn test-storybook --no-cache", "test-storybook:json": "yarn test-storybook --stories-json", "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx serve storybook-static -l 6006\" \"wait-on tcp:6006 && yarn test-storybook\"", diff --git a/stories/expected-failures/Failure.stories.tsx b/stories/expected-failures/Failure.stories.tsx index f8da9231..9f2ea5aa 100644 --- a/stories/expected-failures/Failure.stories.tsx +++ b/stories/expected-failures/Failure.stories.tsx @@ -10,6 +10,11 @@ export default { const Template = (args) => ; +export const ComponentLogsErrors = () => { + console.error('Console error with a failure'); + return
Hello world
; +}; + export const ComponentThrowsErrors = () => { throw new Error('Component has a failure'); };