diff --git a/.storybook/preview.ts b/.storybook/preview.ts index 9cfd9258..a1cc7482 100644 --- a/.storybook/preview.ts +++ b/.storybook/preview.ts @@ -1,3 +1,4 @@ +import type { Preview } from '@storybook/react'; import { isTestRunner } from './is-test-runner'; const withSkippableTests = (StoryFn, { parameters }) => { @@ -8,4 +9,9 @@ const withSkippableTests = (StoryFn, { parameters }) => { return StoryFn(); }; -export const decorators = [withSkippableTests]; +const preview: Preview = { + tags: ['global-tag'], + decorators: [withSkippableTests], +}; + +export default preview; diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts index 11045b57..c1038230 100644 --- a/.storybook/test-runner.ts +++ b/.storybook/test-runner.ts @@ -8,6 +8,9 @@ const skipSnapshots = process.env.SKIP_SNAPSHOTS === 'true'; const config: TestRunnerConfig = { logLevel: 'verbose', + errorMessageFormatter: (message) => { + return message; + }, tags: { exclude: ['exclude'], include: [], diff --git a/README.md b/README.md index 1e208862..42052c10 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ module.exports = { ## Filtering tests (experimental) -You might want to skip certain stories in the test-runner, run tests only against a subset of stories, or exclude certain stories entirely from your tests. This is possible via the `tags` annotation. +You might want to skip certain stories in the test-runner, run tests only against a subset of stories, or exclude certain stories entirely from your tests. This is possible via the `tags` annotation. By default, the test-runner includes every story with the `"test"` tag. This tag is included by default in Storybook 8 for all stories, unless the user tells otherwise via [tag negation](https://storybook.js.org/docs/writing-stories/tags#removing-tags). This annotation can be part of a story, therefore only applying to it, or the component meta (the default export), which applies to all stories in the file: @@ -729,6 +729,23 @@ const config: TestRunnerConfig = { export default config; ``` +#### errorMessageFormatter + +The `errorMessageFormatter` property defines a function that will pre-format the error messages before they get reported in the CLI: + +```ts +// .storybook/test-runner.ts +import type { TestRunnerConfig } from '@storybook/test-runner'; + +const config: TestRunnerConfig = { + errorMessageFormatter: (message) => { + // manipulate the error message as you like + return message; + }, +}; +export default config; +``` + ### Utility functions For more specific use cases, the test runner provides utility functions that could be useful to you. diff --git a/package.json b/package.json index 72aed840..a8659ea0 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "@storybook/csf": "^0.1.2", "@storybook/csf-tools": "^8.0.0", "@storybook/preview-api": "^8.0.0", - "@swc/core": "1.5.7", + "@swc/core": "^1.5.22", "@swc/jest": "^0.2.23", "expect-playwright": "^0.8.0", "jest": "^29.6.4", diff --git a/src/csf/transformCsf.ts b/src/csf/transformCsf.ts index 27633bbc..b8c19cdf 100644 --- a/src/csf/transformCsf.ts +++ b/src/csf/transformCsf.ts @@ -2,7 +2,7 @@ import { loadCsf } from '@storybook/csf-tools'; import * as t from '@babel/types'; import generate from '@babel/generator'; -import { toId, storyNameFromExport } from '@storybook/csf'; +import { toId, storyNameFromExport, combineTags } from '@storybook/csf'; import dedent from 'ts-dedent'; import { getTagOptions } from '../util/getTagOptions'; @@ -108,7 +108,8 @@ export const transformCsf = ( beforeEachPrefixer, insertTestIfEmpty, makeTitle, - }: TransformOptions + previewAnnotations = { tags: [] }, + }: TransformOptions & { previewAnnotations?: Record } ) => { const { includeTags, excludeTags, skipTags } = getTagOptions(); @@ -126,7 +127,14 @@ export const transformCsf = ( acc[key].play = annotations.play; } - acc[key].tags = csf._stories[key].tags || csf.meta?.tags || []; + acc[key].tags = combineTags( + 'test', + 'dev', + ...previewAnnotations.tags, + ...(csf.meta?.tags || []), + ...(csf._stories[key].tags || []) + ); + return acc; }, {} diff --git a/src/playwright/hooks.ts b/src/playwright/hooks.ts index 851bd9c1..2beac9d2 100644 --- a/src/playwright/hooks.ts +++ b/src/playwright/hooks.ts @@ -63,6 +63,11 @@ export interface TestRunnerConfig { * @default 'info' */ logLevel?: 'info' | 'warn' | 'error' | 'verbose' | 'none'; + + /** + * Defines a custom function to process the error message. Useful to sanitize error messages or to add additional information. + */ + errorMessageFormatter?: (error: string) => string; } export const setPreVisit = (preVisit: TestHook) => { diff --git a/src/playwright/transformPlaywright.test.ts b/src/playwright/transformPlaywright.test.ts index 2e43f5f7..4c887fc4 100644 --- a/src/playwright/transformPlaywright.test.ts +++ b/src/playwright/transformPlaywright.test.ts @@ -42,6 +42,7 @@ describe('Playwright', () => { delete process.env.STORYBOOK_INCLUDE_TAGS; delete process.env.STORYBOOK_EXCLUDE_TAGS; delete process.env.STORYBOOK_SKIP_TAGS; + delete process.env.STORYBOOK_PREVIEW_TAGS; }); describe('tag filtering mechanism', () => { @@ -324,14 +325,17 @@ describe('Playwright', () => { `); }); it('should work in conjunction with includeTags, excludeTags and skipTags', () => { - process.env.STORYBOOK_INCLUDE_TAGS = 'play,design'; + process.env.STORYBOOK_INCLUDE_TAGS = 'play,design,global-tag'; process.env.STORYBOOK_SKIP_TAGS = 'skip'; process.env.STORYBOOK_EXCLUDE_TAGS = 'exclude'; + process.env.STORYBOOK_PREVIEW_TAGS = 'global-tag'; + // Should result in: // - A being excluded // - B being included, but skipped // - C being included - // - D being excluded + // - D being included + // - E being excluded expect( transformPlaywright( dedent` @@ -339,7 +343,8 @@ describe('Playwright', () => { export const A = { tags: ['play', 'exclude'] }; export const B = { tags: ['play', 'skip'] }; export const C = { tags: ['design'] }; - export const D = { }; + export const D = { tags: ['global-tag'] }; + export const E = { }; `, filename ) @@ -436,6 +441,275 @@ describe('Playwright', () => { } }); }); + describe("D", () => { + it("smoke-test", async () => { + const testFn = async () => { + const context = { + id: "example-foo-bar--d", + title: "Example/foo/bar", + name: "D" + }; + if (globalThis.__sbPreVisit) { + await globalThis.__sbPreVisit(page, context); + } + const result = await page.evaluate(({ + id, + hasPlayFn + }) => __test(id, hasPlayFn), { + id: "example-foo-bar--d" + }); + if (globalThis.__sbPostVisit) { + await globalThis.__sbPostVisit(page, context); + } + if (globalThis.__sbCollectCoverage) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error(\`[Test runner] An error occurred when evaluating code coverage: + The code in this story is not instrumented, which means the coverage setup is likely not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage\`); + } + await jestPlaywright.saveCoverage(page); + } + return result; + }; + try { + await testFn(); + } catch (err) { + if (err.toString().includes('Execution context was destroyed')) { + console.log(\`An error occurred in the following story, most likely because of a navigation: "\${"Example/foo/bar"}/\${"D"}". Retrying...\`); + await jestPlaywright.resetPage(); + await globalThis.__sbSetupPage(globalThis.page, globalThis.context); + await testFn(); + } else { + throw err; + } + } + }); + }); + describe("E", () => { + it("smoke-test", async () => { + const testFn = async () => { + const context = { + id: "example-foo-bar--e", + title: "Example/foo/bar", + name: "E" + }; + if (globalThis.__sbPreVisit) { + await globalThis.__sbPreVisit(page, context); + } + const result = await page.evaluate(({ + id, + hasPlayFn + }) => __test(id, hasPlayFn), { + id: "example-foo-bar--e" + }); + if (globalThis.__sbPostVisit) { + await globalThis.__sbPostVisit(page, context); + } + if (globalThis.__sbCollectCoverage) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error(\`[Test runner] An error occurred when evaluating code coverage: + The code in this story is not instrumented, which means the coverage setup is likely not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage\`); + } + await jestPlaywright.saveCoverage(page); + } + return result; + }; + try { + await testFn(); + } catch (err) { + if (err.toString().includes('Execution context was destroyed')) { + console.log(\`An error occurred in the following story, most likely because of a navigation: "\${"Example/foo/bar"}/\${"E"}". Retrying...\`); + await jestPlaywright.resetPage(); + await globalThis.__sbSetupPage(globalThis.page, globalThis.context); + await testFn(); + } else { + throw err; + } + } + }); + }); + }); + } + `); + }); + it('should work with tag negation', () => { + process.env.STORYBOOK_INCLUDE_TAGS = 'play,test'; + process.env.STORYBOOK_PREVIEW_TAGS = '!test'; + // Should result in: + // - A being included + // - B being excluded because it has no play nor test tag (removed by negation in preview tags) + // - C being included because it has test tag (overwritten via story tags) + expect( + transformPlaywright( + dedent` + export default { title: 'foo/bar', component: Button, tags: ['play'] }; + export const A = { }; + export const B = { tags: ['!play'] }; + export const C = { tags: ['!play', 'test'] }; + `, + filename + ) + ).toMatchInlineSnapshot(` + if (!require.main) { + describe("Example/foo/bar", () => { + describe("A", () => { + it("smoke-test", async () => { + const testFn = async () => { + const context = { + id: "example-foo-bar--a", + title: "Example/foo/bar", + name: "A" + }; + if (globalThis.__sbPreVisit) { + await globalThis.__sbPreVisit(page, context); + } + const result = await page.evaluate(({ + id, + hasPlayFn + }) => __test(id, hasPlayFn), { + id: "example-foo-bar--a" + }); + if (globalThis.__sbPostVisit) { + await globalThis.__sbPostVisit(page, context); + } + if (globalThis.__sbCollectCoverage) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error(\`[Test runner] An error occurred when evaluating code coverage: + The code in this story is not instrumented, which means the coverage setup is likely not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage\`); + } + await jestPlaywright.saveCoverage(page); + } + return result; + }; + try { + await testFn(); + } catch (err) { + if (err.toString().includes('Execution context was destroyed')) { + console.log(\`An error occurred in the following story, most likely because of a navigation: "\${"Example/foo/bar"}/\${"A"}". Retrying...\`); + await jestPlaywright.resetPage(); + await globalThis.__sbSetupPage(globalThis.page, globalThis.context); + await testFn(); + } else { + throw err; + } + } + }); + }); + describe("C", () => { + it("smoke-test", async () => { + const testFn = async () => { + const context = { + id: "example-foo-bar--c", + title: "Example/foo/bar", + name: "C" + }; + if (globalThis.__sbPreVisit) { + await globalThis.__sbPreVisit(page, context); + } + const result = await page.evaluate(({ + id, + hasPlayFn + }) => __test(id, hasPlayFn), { + id: "example-foo-bar--c" + }); + if (globalThis.__sbPostVisit) { + await globalThis.__sbPostVisit(page, context); + } + if (globalThis.__sbCollectCoverage) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error(\`[Test runner] An error occurred when evaluating code coverage: + The code in this story is not instrumented, which means the coverage setup is likely not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage\`); + } + await jestPlaywright.saveCoverage(page); + } + return result; + }; + try { + await testFn(); + } catch (err) { + if (err.toString().includes('Execution context was destroyed')) { + console.log(\`An error occurred in the following story, most likely because of a navigation: "\${"Example/foo/bar"}/\${"C"}". Retrying...\`); + await jestPlaywright.resetPage(); + await globalThis.__sbSetupPage(globalThis.page, globalThis.context); + await testFn(); + } else { + throw err; + } + } + }); + }); + }); + } + `); + }); + it('should include "test" tag by default', () => { + // Should result in: + // - A being included + // - B being excluded + expect( + transformPlaywright( + dedent` + export default { title: 'foo/bar', component: Button }; + export const A = { }; + export const B = { tags: ['!test'] }; + `, + filename + ) + ).toMatchInlineSnapshot(` + if (!require.main) { + describe("Example/foo/bar", () => { + describe("A", () => { + it("smoke-test", async () => { + const testFn = async () => { + const context = { + id: "example-foo-bar--a", + title: "Example/foo/bar", + name: "A" + }; + if (globalThis.__sbPreVisit) { + await globalThis.__sbPreVisit(page, context); + } + const result = await page.evaluate(({ + id, + hasPlayFn + }) => __test(id, hasPlayFn), { + id: "example-foo-bar--a" + }); + if (globalThis.__sbPostVisit) { + await globalThis.__sbPostVisit(page, context); + } + if (globalThis.__sbCollectCoverage) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error(\`[Test runner] An error occurred when evaluating code coverage: + The code in this story is not instrumented, which means the coverage setup is likely not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage\`); + } + await jestPlaywright.saveCoverage(page); + } + return result; + }; + try { + await testFn(); + } catch (err) { + if (err.toString().includes('Execution context was destroyed')) { + console.log(\`An error occurred in the following story, most likely because of a navigation: "\${"Example/foo/bar"}/\${"A"}". Retrying...\`); + await jestPlaywright.resetPage(); + await globalThis.__sbSetupPage(globalThis.page, globalThis.context); + await testFn(); + } else { + throw err; + } + } + }); + }); }); } `); diff --git a/src/playwright/transformPlaywright.ts b/src/playwright/transformPlaywright.ts index f74699ae..fce02b79 100644 --- a/src/playwright/transformPlaywright.ts +++ b/src/playwright/transformPlaywright.ts @@ -75,11 +75,13 @@ const makeTitleFactory = (filename: string) => { }; export const transformPlaywright = (src: string, filename: string) => { + const tags = process.env.STORYBOOK_PREVIEW_TAGS?.split(',') ?? []; const transformOptions = { testPrefixer, insertTestIfEmpty: true, clearBody: true, makeTitle: makeTitleFactory(filename), + previewAnnotations: { tags }, }; const result = transformCsf(src, transformOptions); diff --git a/src/playwright/transformPlaywrightJson.test.ts b/src/playwright/transformPlaywrightJson.test.ts index 2d16d735..e3712d82 100644 --- a/src/playwright/transformPlaywrightJson.test.ts +++ b/src/playwright/transformPlaywrightJson.test.ts @@ -25,17 +25,19 @@ describe('Playwright Json', () => { id: 'example-header--logged-in', title: 'Example/Header', name: 'Logged In', - tags: ['play-fn'], + tags: ['test', 'play-fn'], }, 'example-header--logged-out': { id: 'example-header--logged-out', title: 'Example/Header', name: 'Logged Out', + tags: ['test'], }, 'example-page--logged-in': { id: 'example-page--logged-in', title: 'Example/Page', name: 'Logged In', + tags: ['test'], }, }, } satisfies V4Index; @@ -656,6 +658,76 @@ describe('Playwright Json', () => { } `); }); + + it('should include "test" tag by default', () => { + process.env.STORYBOOK_INCLUDE_TAGS = 'test'; + const input = { + v: 3, + stories: { + 'example-page--logged-in': { + id: 'example-page--logged-in', + title: 'Example/Page', + name: 'Logged In', + parameters: { + __id: 'example-page--logged-in', + docsOnly: false, + fileName: './stories/basic/Page.stories.js', + }, + }, + }, + } satisfies V3StoriesIndex; + expect(transformPlaywrightJson(input)).toMatchInlineSnapshot(` + { + "example-page": "describe("Example/Page", () => { + describe("Logged In", () => { + it("smoke-test", async () => { + const testFn = async () => { + const context = { + id: "example-page--logged-in", + title: "Example/Page", + name: "Logged In" + }; + if (globalThis.__sbPreVisit) { + await globalThis.__sbPreVisit(page, context); + } + const result = await page.evaluate(({ + id, + hasPlayFn + }) => __test(id, hasPlayFn), { + id: "example-page--logged-in" + }); + if (globalThis.__sbPostVisit) { + await globalThis.__sbPostVisit(page, context); + } + if (globalThis.__sbCollectCoverage) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error(\`[Test runner] An error occurred when evaluating code coverage: + The code in this story is not instrumented, which means the coverage setup is likely not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage\`); + } + await jestPlaywright.saveCoverage(page); + } + return result; + }; + try { + await testFn(); + } catch (err) { + if (err.toString().includes('Execution context was destroyed')) { + console.log(\`An error occurred in the following story, most likely because of a navigation: "\${"Example/Page"}/\${"Logged In"}". Retrying...\`); + await jestPlaywright.resetPage(); + await globalThis.__sbSetupPage(globalThis.page, globalThis.context); + await testFn(); + } else { + throw err; + } + } + }); + }); + });", + } + `); + }); }); }); diff --git a/src/playwright/transformPlaywrightJson.ts b/src/playwright/transformPlaywrightJson.ts index c22bea24..1777c85a 100644 --- a/src/playwright/transformPlaywrightJson.ts +++ b/src/playwright/transformPlaywrightJson.ts @@ -59,6 +59,11 @@ export const makeDescribe = (title: string, stmts: t.Statement[]) => { ); }; +type V3Story = Omit & { parameters?: StoryParameters }; +export type V3StoriesIndex = { + v: 3; + stories: Record; +}; type V4Entry = { type?: 'story' | 'docs'; id: StoryId; @@ -71,17 +76,18 @@ export type V4Index = { entries: Record; }; +type V5Entry = V4Entry & { tags: string[] }; +export type V5Index = { + v: 5; + entries: Record; +}; + type StoryParameters = { __id: StoryId; docsOnly?: boolean; fileName?: string; }; -type V3Story = Omit & { parameters?: StoryParameters }; -export type V3StoriesIndex = { - v: 3; - stories: Record; -}; export type UnsupportedVersion = { v: number }; const isV3DocsOnly = (stories: V3Story[]) => stories.length === 1 && stories[0].name === 'Page'; @@ -93,6 +99,7 @@ function v3TitleMapToV4TitleMap(titleIdToStories: Record) { ({ parameters, ...story }) => ({ type: isV3DocsOnly(stories) ? 'docs' : 'story', + tags: isV3DocsOnly(stories) ? [] : ['test', 'dev'], ...story, }) satisfies V4Entry ), @@ -100,6 +107,26 @@ function v3TitleMapToV4TitleMap(titleIdToStories: Record) { ); } +/** + * Storybook 8.0 and below did not automatically tag stories with 'dev'. + * Therefore Storybook 8.1 and above would not show composed 8.0 stories by default. + * This function adds the 'dev' tag to all stories in the index to workaround this issue. + */ +function v4TitleMapToV5TitleMap(titleIdToStories: Record) { + return Object.fromEntries( + Object.entries(titleIdToStories).map(([id, stories]) => [ + id, + stories.map( + (story) => + ({ + ...story, + tags: story.tags ? ['test', 'dev', ...story.tags] : ['test', 'dev'], + }) satisfies V4Entry + ), + ]) + ); +} + function groupByTitleId(entries: T[]) { return entries.reduce>((acc, entry) => { const titleId = toId(entry.title); @@ -120,10 +147,12 @@ export const transformPlaywrightJson = (index: V3StoriesIndex | V4Index | Unsupp Object.values((index as V3StoriesIndex).stories) ); titleIdToEntries = v3TitleMapToV4TitleMap(titleIdToStories); - // v4 and v5 are pretty much similar, so we process it in the same way - } else if (index.v === 4 || index.v === 5) { + } else if (index.v === 4) { // TODO: Once Storybook 8.0 is released, we should only support v4 and higher titleIdToEntries = groupByTitleId(Object.values((index as V4Index).entries)); + titleIdToEntries = v4TitleMapToV5TitleMap(titleIdToEntries); + } else if (index.v === 5) { + titleIdToEntries = groupByTitleId(Object.values((index as V4Index).entries)); } else { throw new Error(`Unsupported version ${index.v}`); } diff --git a/src/setup-page-script.ts b/src/setup-page-script.ts index 8d293163..e1458cdc 100644 --- a/src/setup-page-script.ts +++ b/src/setup-page-script.ts @@ -30,7 +30,8 @@ const TEST_RUNNER_DEBUG_PRINT_LIMIT = parseInt('{{debugPrintLimit}}', 10); // Type definitions for globals declare global { // this is defined in setup-page.ts and can be used for logging from the browser to node, helpful for debugging - var logToPage: (message: string) => void; + var logToPage: (message: string) => Promise; + var testRunner_errorMessageFormatter: (message: string) => Promise; } // Type definitions for function parameters and return types @@ -205,21 +206,39 @@ function addToUserAgent(extra: string): void { // Custom error class class StorybookTestRunnerError extends Error { - constructor(storyId: string, errorMessage: string, logs: string[] = []) { - super(errorMessage); + constructor( + storyId: string, + errorMessage: string, + logs: string[] = [], + isMessageFormatted: boolean = false + ) { + const message = isMessageFormatted + ? errorMessage + : StorybookTestRunnerError.buildErrorMessage(storyId, errorMessage, logs); + super(message); + this.name = 'StorybookTestRunnerError'; + } + + public static buildErrorMessage( + storyId: string, + errorMessage: string, + logs: string[] = [] + ): string { const storyUrl = `${TEST_RUNNER_STORYBOOK_URL}?path=/story/${storyId}`; const finalStoryUrl = `${storyUrl}&addonPanel=storybook/interactions/panel`; const separator = '\n\n--------------------------------------------------'; // The original error message will also be collected in the logs, so we filter it to avoid duplication - const finalLogs = logs.filter((err) => !err.includes(errorMessage)); + const finalLogs = logs.filter((err: string) => !err.includes(errorMessage)); const extraLogs = finalLogs.length > 0 ? separator + '\n\nBrowser logs:\n\n' + finalLogs.join('\n\n') : ''; - this.message = `\nAn error occurred in the following story. Access the link for full output:\n${finalStoryUrl}\n\nMessage:\n ${truncate( + const message = `\nAn error occurred in the following story. Access the link for full output:\n${finalStoryUrl}\n\nMessage:\n ${truncate( errorMessage, TEST_RUNNER_DEBUG_PRINT_LIMIT )}\n${extraLogs}`; + + return message; } } @@ -351,13 +370,32 @@ async function __test(storyId: string): Promise { }; return new Promise((resolve, reject) => { + const rejectWithFormattedError = (storyId: string, message: string) => { + const errorMessage = StorybookTestRunnerError.buildErrorMessage(storyId, message, logs); + + testRunner_errorMessageFormatter(errorMessage) + .then((formattedMessage) => { + reject(new StorybookTestRunnerError(storyId, formattedMessage, logs, true)); + }) + .catch((error) => { + reject( + new StorybookTestRunnerError( + storyId, + 'There was an error when executing the errorMessageFormatter defiend in your Storybook test-runner config file. Please fix it and rerun the tests:\n\n' + + error.message + ) + ); + }); + }; + const listeners = { [TEST_RUNNER_RENDERED_EVENT]: () => { cleanup(listeners); if (hasErrors) { - reject(new StorybookTestRunnerError(storyId, 'Browser console errors', logs)); + rejectWithFormattedError(storyId, 'Browser console errors'); + } else { + resolve(document.getElementById('root')); } - resolve(document.getElementById('root')); }, storyUnchanged: () => { @@ -367,34 +405,29 @@ async function __test(storyId: string): Promise { storyErrored: ({ description }: { description: string }) => { cleanup(listeners); - reject(new StorybookTestRunnerError(storyId, description, logs)); + rejectWithFormattedError(storyId, description); }, storyThrewException: (error: Error) => { cleanup(listeners); - reject(new StorybookTestRunnerError(storyId, error.message, logs)); + rejectWithFormattedError(storyId, error.message); }, playFunctionThrewException: (error: Error) => { cleanup(listeners); - reject(new StorybookTestRunnerError(storyId, error.message, logs)); + + rejectWithFormattedError(storyId, error.message); }, unhandledErrorsWhilePlaying: ([error]: Error[]) => { cleanup(listeners); - reject(new StorybookTestRunnerError(storyId, error.message, logs)); + rejectWithFormattedError(storyId, error.message); }, storyMissing: (id: string) => { cleanup(listeners); if (id === storyId) { - reject( - new StorybookTestRunnerError( - storyId, - 'The story was missing when trying to access it.', - logs - ) - ); + rejectWithFormattedError(storyId, 'The story was missing when trying to access it.'); } }, }; diff --git a/src/setup-page.ts b/src/setup-page.ts index 1edb92d8..c96394ec 100644 --- a/src/setup-page.ts +++ b/src/setup-page.ts @@ -61,6 +61,14 @@ export const setupPage = async (page: Page, browserContext: BrowserContext) => { // if we ever want to log something from the browser to node await page.exposeBinding('logToPage', (_, message) => console.log(message)); + await page.exposeBinding('testRunner_errorMessageFormatter', (_, message: string) => { + if (testRunnerConfig.errorMessageFormatter) { + return testRunnerConfig.errorMessageFormatter(message); + } + + return message; + }); + const finalStorybookUrl = referenceURL ?? targetURL ?? ''; const testRunnerPackageLocation = await pkgUp({ cwd: __dirname }); if (!testRunnerPackageLocation) throw new Error('Could not find test-runner package location'); diff --git a/src/test-storybook.ts b/src/test-storybook.ts index b846f8cd..1d78deea 100644 --- a/src/test-storybook.ts +++ b/src/test-storybook.ts @@ -5,7 +5,7 @@ import { execSync } from 'child_process'; import fetch from 'node-fetch'; import canBindToHost from 'can-bind-to-host'; import dedent from 'ts-dedent'; -import path from 'path'; +import path, { join, resolve } from 'path'; import tempy from 'tempy'; import { JestOptions, getCliOptions } from './util/getCliOptions'; @@ -15,6 +15,8 @@ import { transformPlaywrightJson } from './playwright/transformPlaywrightJson'; import { glob } from 'glob'; import { TestRunnerConfig } from './playwright/hooks'; +import { getInterpretedFile } from '@storybook/core-common'; +import { readConfig } from '@storybook/csf-tools'; // Do this as the first thing so that any code reading it knows the right env. process.env.BABEL_ENV = 'test'; @@ -243,8 +245,10 @@ function ejectConfiguration() { \n`); } - fs.copyFileSync(origin, destination); - log('Configuration file successfully copied as test-runner-jest.config.js'); + // copy contents of origin and replace ../dist with @storybook/test-runner + const content = fs.readFileSync(origin, 'utf-8').replace(/..\/dist/g, '@storybook/test-runner'); + fs.writeFileSync(destination, content); + log(`Configuration file successfully generated at ${destination}`); } function warnOnce(message: string) { @@ -258,6 +262,16 @@ function warnOnce(message: string) { }; } +const extractTagsFromPreview = async (configDir = '.storybook') => { + const previewConfigPath = getInterpretedFile(join(resolve(configDir), 'preview')); + + if (!previewConfigPath) return []; + const previewConfig = await readConfig(previewConfigPath); + const tags = previewConfig.getFieldValue(['tags']) ?? []; + + return tags.join(','); +}; + const main = async () => { const { jestOptions, runnerOptions } = getCliOptions(); @@ -366,6 +380,10 @@ const main = async () => { const { storiesPaths, lazyCompilation } = getStorybookMetadata(); process.env.STORYBOOK_STORIES_PATTERN = storiesPaths; + // 1 - We extract tags from preview file statically like it's done by the Storybook indexer. We only do this in non-index-json mode because it's not needed in that mode + // 2 - We pass it via env variable to avoid having to use async code in the babel plugin + process.env.STORYBOOK_PREVIEW_TAGS = await extractTagsFromPreview(runnerOptions.configDir); + if (lazyCompilation && isLocalStorybookIp) { log( `You're running Storybook with lazy compilation enabled, and will likely cause issues with the test runner locally. Consider disabling 'lazyCompilation' in ${runnerOptions.configDir}/main.js when running 'test-storybook' locally.` diff --git a/src/util/getTagOptions.ts b/src/util/getTagOptions.ts index 1bbdad8c..0a72d942 100644 --- a/src/util/getTagOptions.ts +++ b/src/util/getTagOptions.ts @@ -15,7 +15,7 @@ export function getTagOptions() { const config = getTestRunnerConfig(); let tagOptions = { - includeTags: config?.tags?.include || [], + includeTags: config?.tags?.include || ['test'], excludeTags: config?.tags?.exclude || [], skipTags: config?.tags?.skip || [], } as TagOptions; diff --git a/yarn.lock b/yarn.lock index 00effd1e..1026b4ba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -437,10 +437,10 @@ __metadata: languageName: node linkType: hard -"@babel/helper-string-parser@npm:^7.24.1": - version: 7.24.1 - resolution: "@babel/helper-string-parser@npm:7.24.1" - checksum: 8404e865b06013979a12406aab4c0e8d2e377199deec09dfe9f57b833b0c9ce7b6e8c1c553f2da8d0bcd240c5005bd7a269f4fef0d628aeb7d5fe035c436fb67 +"@babel/helper-string-parser@npm:^7.24.7": + version: 7.24.7 + resolution: "@babel/helper-string-parser@npm:7.24.7" + checksum: 09568193044a578743dd44bf7397940c27ea693f9812d24acb700890636b376847a611cdd0393a928544e79d7ad5b8b916bd8e6e772bc8a10c48a647a96e7b1a languageName: node linkType: hard @@ -451,6 +451,13 @@ __metadata: languageName: node linkType: hard +"@babel/helper-validator-identifier@npm:^7.24.7": + version: 7.24.7 + resolution: "@babel/helper-validator-identifier@npm:7.24.7" + checksum: 6799ab117cefc0ecd35cd0b40ead320c621a298ecac88686a14cffceaac89d80cdb3c178f969861bf5fa5e4f766648f9161ea0752ecfe080d8e89e3147270257 + languageName: node + linkType: hard + "@babel/helper-validator-option@npm:^7.23.5": version: 7.23.5 resolution: "@babel/helper-validator-option@npm:7.23.5" @@ -1686,13 +1693,13 @@ __metadata: linkType: hard "@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.15, @babel/types@npm:^7.22.5, @babel/types@npm:^7.23.0, @babel/types@npm:^7.23.4, @babel/types@npm:^7.24.0, @babel/types@npm:^7.24.5, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": - version: 7.24.5 - resolution: "@babel/types@npm:7.24.5" + version: 7.24.7 + resolution: "@babel/types@npm:7.24.7" dependencies: - "@babel/helper-string-parser": ^7.24.1 - "@babel/helper-validator-identifier": ^7.24.5 + "@babel/helper-string-parser": ^7.24.7 + "@babel/helper-validator-identifier": ^7.24.7 to-fast-properties: ^2.0.0 - checksum: 8eeeacd996593b176e649ee49d8dc3f26f9bb6aa1e3b592030e61a0e58ea010fb018dccc51e5314c8139409ea6cbab02e29b33e674e1f6962d8e24c52da6375b + checksum: 3e4437fced97e02982972ce5bebd318c47d42c9be2152c0fd28c6f786cc74086cc0a8fb83b602b846e41df37f22c36254338eada1a47ef9d8a1ec92332ca3ea8 languageName: node linkType: hard @@ -3970,7 +3977,7 @@ __metadata: "@storybook/react": ^8.0.0 "@storybook/react-vite": ^8.0.0 "@storybook/test": ^8.0.0 - "@swc/core": 1.5.7 + "@swc/core": ^1.5.22 "@swc/jest": ^0.2.23 "@types/jest": ^29.0.0 "@types/node": ^16.4.1 @@ -4066,94 +4073,94 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-darwin-arm64@npm:1.5.7" +"@swc/core-darwin-arm64@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-darwin-arm64@npm:1.5.27" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-darwin-x64@npm:1.5.7" +"@swc/core-darwin-x64@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-darwin-x64@npm:1.5.27" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.5.7" +"@swc/core-linux-arm-gnueabihf@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.5.27" conditions: os=linux & cpu=arm languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-linux-arm64-gnu@npm:1.5.7" +"@swc/core-linux-arm64-gnu@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-linux-arm64-gnu@npm:1.5.27" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-linux-arm64-musl@npm:1.5.7" +"@swc/core-linux-arm64-musl@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-linux-arm64-musl@npm:1.5.27" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-linux-x64-gnu@npm:1.5.7" +"@swc/core-linux-x64-gnu@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-linux-x64-gnu@npm:1.5.27" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-linux-x64-musl@npm:1.5.7" +"@swc/core-linux-x64-musl@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-linux-x64-musl@npm:1.5.27" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-win32-arm64-msvc@npm:1.5.7" +"@swc/core-win32-arm64-msvc@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-win32-arm64-msvc@npm:1.5.27" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-win32-ia32-msvc@npm:1.5.7" +"@swc/core-win32-ia32-msvc@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-win32-ia32-msvc@npm:1.5.27" conditions: os=win32 & cpu=ia32 languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core-win32-x64-msvc@npm:1.5.7" +"@swc/core-win32-x64-msvc@npm:1.5.27": + version: 1.5.27 + resolution: "@swc/core-win32-x64-msvc@npm:1.5.27" conditions: os=win32 & cpu=x64 languageName: node linkType: hard -"@swc/core@npm:1.5.7": - version: 1.5.7 - resolution: "@swc/core@npm:1.5.7" +"@swc/core@npm:^1.5.22": + version: 1.5.27 + resolution: "@swc/core@npm:1.5.27" dependencies: - "@swc/core-darwin-arm64": 1.5.7 - "@swc/core-darwin-x64": 1.5.7 - "@swc/core-linux-arm-gnueabihf": 1.5.7 - "@swc/core-linux-arm64-gnu": 1.5.7 - "@swc/core-linux-arm64-musl": 1.5.7 - "@swc/core-linux-x64-gnu": 1.5.7 - "@swc/core-linux-x64-musl": 1.5.7 - "@swc/core-win32-arm64-msvc": 1.5.7 - "@swc/core-win32-ia32-msvc": 1.5.7 - "@swc/core-win32-x64-msvc": 1.5.7 - "@swc/counter": ^0.1.2 - "@swc/types": 0.1.7 + "@swc/core-darwin-arm64": 1.5.27 + "@swc/core-darwin-x64": 1.5.27 + "@swc/core-linux-arm-gnueabihf": 1.5.27 + "@swc/core-linux-arm64-gnu": 1.5.27 + "@swc/core-linux-arm64-musl": 1.5.27 + "@swc/core-linux-x64-gnu": 1.5.27 + "@swc/core-linux-x64-musl": 1.5.27 + "@swc/core-win32-arm64-msvc": 1.5.27 + "@swc/core-win32-ia32-msvc": 1.5.27 + "@swc/core-win32-x64-msvc": 1.5.27 + "@swc/counter": ^0.1.3 + "@swc/types": ^0.1.8 peerDependencies: - "@swc/helpers": ^0.5.0 + "@swc/helpers": "*" dependenciesMeta: "@swc/core-darwin-arm64": optional: true @@ -4178,11 +4185,11 @@ __metadata: peerDependenciesMeta: "@swc/helpers": optional: true - checksum: 8e11626b782df914ee53dcb3e7f52e4bd2e1a896873c0e76ec674d19d05d87eec06e2223e0958d68ef1e0cdfb4cd505e3b1a297561e9506063738337f0c5409d + checksum: a7082899f92efd623a31b225f1571019fa02d316714d9f33c5ecc793c4000a47b7e062571be70a6f0ecb3d8781776198b3bb4107b466ed9c19b72b7012cc9d04 languageName: node linkType: hard -"@swc/counter@npm:^0.1.2, @swc/counter@npm:^0.1.3": +"@swc/counter@npm:^0.1.3": version: 0.1.3 resolution: "@swc/counter@npm:0.1.3" checksum: df8f9cfba9904d3d60f511664c70d23bb323b3a0803ec9890f60133954173047ba9bdeabce28cd70ba89ccd3fd6c71c7b0bd58be85f611e1ffbe5d5c18616598 @@ -4202,12 +4209,12 @@ __metadata: languageName: node linkType: hard -"@swc/types@npm:0.1.7": - version: 0.1.7 - resolution: "@swc/types@npm:0.1.7" +"@swc/types@npm:^0.1.8": + version: 0.1.8 + resolution: "@swc/types@npm:0.1.8" dependencies: "@swc/counter": ^0.1.3 - checksum: e251f6994de12a2a81ed79d902a521398feda346022e09567c758eee1cca606743c9bb296de74d6fbe339f953eaf69176202babc8ef9c911d5d538fc0790df28 + checksum: e564d0e37b0e28546973c6d50c7a179395912a97168d695cfe9cf1051199c8b828680cdafcb8d43948f76d3703873bafb88dfb5bc2dfe0596b4ad18fcaf90c80 languageName: node linkType: hard