Skip to content

Commit

Permalink
fix(v2): Fix MDX docs being considered as partials when siteDir match…
Browse files Browse the repository at this point in the history
… the _ prefix convention (#5199)

* Add _ to dogfood docs folder to cover against edge case

* Fix edge case with MDX partials when site / content dir contains a _ prefix

* add globUtils tests

* proper dogfooding folder re-organization, all content plugins being used

* refactor dogfooding folder + expose /tests page index

* fix page plugin ignoring options.routeBasePath
  • Loading branch information
slorber authored Jul 21, 2021
1 parent a272912 commit 4d06f26
Show file tree
Hide file tree
Showing 28 changed files with 271 additions and 54 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/v2-build-size-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
with:
repo-token: '${{ secrets.GITHUB_TOKEN }}'
build-script: 'build:v2:en'
pattern: '{website/build/assets/js/main*js,website/build/assets/css/styles*css,website/.docusaurus/globalData.json,website/build/index.html,website/build/blog/index.html,website/build/blog/**/introducing-docusaurus/*,website/build/docs/index.html,website/build/docs/installation/index.html,website/build/docs-tests/index.html,website/build/docs-tests/standalone/index.html}'
pattern: '{website/build/assets/js/main*js,website/build/assets/css/styles*css,website/.docusaurus/globalData.json,website/build/index.html,website/build/blog/index.html,website/build/blog/**/introducing-docusaurus/*,website/build/docs/index.html,website/build/docs/installation/index.html,website/build/tests/docs/index.html,website/build/tests/docs/standalone/index.html}'
strip-hash: '\.([^;]\w{7})\.'
minimum-change-threshold: 30
compression: 'none'
1 change: 1 addition & 0 deletions packages/docusaurus-mdx-loader/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"@docusaurus/utils": "2.0.0-beta.3",
"@mdx-js/mdx": "^1.6.21",
"@mdx-js/react": "^1.6.21",
"chalk": "^4.1.1",
"escape-html": "^1.0.3",
"file-loader": "^6.2.0",
"fs-extra": "^10.0.0",
Expand Down
19 changes: 15 additions & 4 deletions packages/docusaurus-mdx-loader/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

const {readFile} = require('fs-extra');
const mdx = require('@mdx-js/mdx');
const chalk = require('chalk');
const emoji = require('remark-emoji');
const {
parseFrontMatter,
Expand Down Expand Up @@ -69,8 +70,8 @@ module.exports = async function docusaurusMdxLoader(fileString) {
],
filepath: filePath,
};
let result;

let result;
try {
result = await mdx(content, options);
} catch (err) {
Expand All @@ -90,9 +91,19 @@ module.exports = async function docusaurusMdxLoader(fileString) {
: false;

if (isMDXPartial && hasFrontMatter) {
return callback(
new Error(`MDX partial should not contain FrontMatter: ${filePath}`),
);
const errorMessage = `Docusaurus MDX partial files should not contain FrontMatter.
Those partial files use the _ prefix as a convention by default, but this is configurable.
File at ${filePath} contains FrontMatter that will be ignored: \n${JSON.stringify(
frontMatter,
null,
2,
)}`;
const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
if (shouldError) {
return callback(new Error(errorMessage));
} else {
console.warn(chalk.yellow(errorMessage));
}
}

if (!isMDXPartial) {
Expand Down
10 changes: 7 additions & 3 deletions packages/docusaurus-plugin-content-blog/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import {
reportMessage,
posixPath,
addTrailingPathSeparator,
createMatcher,
createAbsoluteFilePathMatcher,
} from '@docusaurus/utils';
import {
STATIC_DIR_NAME,
Expand Down Expand Up @@ -437,6 +437,7 @@ export default function pluginContentBlog(
},
};

const contentDirs = getContentPathList(contentPaths);
return {
resolve: {
alias: {
Expand All @@ -447,7 +448,7 @@ export default function pluginContentBlog(
rules: [
{
test: /(\.mdx?)$/,
include: getContentPathList(contentPaths)
include: contentDirs
// Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
.map(addTrailingPathSeparator),
use: [
Expand All @@ -460,7 +461,10 @@ export default function pluginContentBlog(
beforeDefaultRemarkPlugins,
beforeDefaultRehypePlugins,
staticDir: path.join(siteDir, STATIC_DIR_NAME),
isMDXPartial: createMatcher(options.exclude),
isMDXPartial: createAbsoluteFilePathMatcher(
options.exclude,
contentDirs,
),
metadataPath: (mdxPath: string) => {
// Note that metadataPath must be the same/in-sync as
// the path from createData for each MDX.
Expand Down
10 changes: 7 additions & 3 deletions packages/docusaurus-plugin-content-docs/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import {
reportMessage,
posixPath,
addTrailingPathSeparator,
createMatcher,
createAbsoluteFilePathMatcher,
} from '@docusaurus/utils';
import {LoadContext, Plugin, RouteConfig} from '@docusaurus/types';
import {loadSidebars, createSidebarsUtils, processSidebars} from './sidebars';
Expand Down Expand Up @@ -394,9 +394,10 @@ export default function pluginContentDocs(
};

function createMDXLoaderRule(): RuleSetRule {
const contentDirs = flatten(versionsMetadata.map(getDocsDirPaths));
return {
test: /(\.mdx?)$/,
include: flatten(versionsMetadata.map(getDocsDirPaths))
include: contentDirs
// Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
.map(addTrailingPathSeparator),
use: compact([
Expand All @@ -409,7 +410,10 @@ export default function pluginContentDocs(
beforeDefaultRehypePlugins,
beforeDefaultRemarkPlugins,
staticDir: path.join(siteDir, STATIC_DIR_NAME),
isMDXPartial: createMatcher(options.exclude),
isMDXPartial: createAbsoluteFilePathMatcher(
options.exclude,
contentDirs,
),
metadataPath: (mdxPath: string) => {
// Note that metadataPath must be the same/in-sync as
// the path from createData for each MDX.
Expand Down
18 changes: 13 additions & 5 deletions packages/docusaurus-plugin-content-pages/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ import {
getFolderContainingFile,
addTrailingPathSeparator,
Globby,
createMatcher,
createAbsoluteFilePathMatcher,
normalizeUrl,
} from '@docusaurus/utils';
import {
LoadContext,
Expand Down Expand Up @@ -124,8 +125,11 @@ export default function pluginContentPages(

const source = path.join(contentPath, relativeSource);
const aliasedSourcePath = aliasedSitePath(source, siteDir);
const pathName = encodePath(fileToPath(relativeSource));
const permalink = pathName.replace(/^\//, baseUrl || '');
const permalink = normalizeUrl([
baseUrl,
options.routeBasePath,
encodePath(fileToPath(relativeSource)),
]);
if (isMarkdownSource(relativeSource)) {
return {
type: 'mdx',
Expand Down Expand Up @@ -194,6 +198,7 @@ export default function pluginContentPages(
beforeDefaultRehypePlugins,
beforeDefaultRemarkPlugins,
} = options;
const contentDirs = getContentPathList(contentPaths);
return {
resolve: {
alias: {
Expand All @@ -204,7 +209,7 @@ export default function pluginContentPages(
rules: [
{
test: /(\.mdx?)$/,
include: getContentPathList(contentPaths)
include: contentDirs
// Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970
.map(addTrailingPathSeparator),
use: [
Expand All @@ -217,7 +222,10 @@ export default function pluginContentPages(
beforeDefaultRehypePlugins,
beforeDefaultRemarkPlugins,
staticDir: path.join(siteDir, STATIC_DIR_NAME),
isMDXPartial: createMatcher(options.exclude),
isMDXPartial: createAbsoluteFilePathMatcher(
options.exclude,
contentDirs,
),
metadataPath: (mdxPath: string) => {
// Note that metadataPath must be the same/in-sync as
// the path from createData for each MDX.
Expand Down
109 changes: 109 additions & 0 deletions packages/docusaurus-utils/src/__tests__/globUtils.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

import {
GlobExcludeDefault,
createMatcher,
createAbsoluteFilePathMatcher,
} from '../globUtils';

describe('createMatcher', () => {
const matcher = createMatcher(GlobExcludeDefault);

test('match default exclude MD/MDX partials correctly', () => {
expect(matcher('doc.md')).toEqual(false);
expect(matcher('category/doc.md')).toEqual(false);
expect(matcher('category/subcategory/doc.md')).toEqual(false);
//
expect(matcher('doc.mdx')).toEqual(false);
expect(matcher('category/doc.mdx')).toEqual(false);
expect(matcher('category/subcategory/doc.mdx')).toEqual(false);
//
expect(matcher('_doc.md')).toEqual(true);
expect(matcher('category/_doc.md')).toEqual(true);
expect(matcher('category/subcategory/_doc.md')).toEqual(true);
expect(matcher('_category/doc.md')).toEqual(true);
expect(matcher('_category/subcategory/doc.md')).toEqual(true);
expect(matcher('category/_subcategory/doc.md')).toEqual(true);
});

test('match default exclude tests correctly', () => {
expect(matcher('xyz.js')).toEqual(false);
expect(matcher('xyz.ts')).toEqual(false);
expect(matcher('xyz.jsx')).toEqual(false);
expect(matcher('xyz.tsx')).toEqual(false);
expect(matcher('folder/xyz.js')).toEqual(false);
expect(matcher('folder/xyz.ts')).toEqual(false);
expect(matcher('folder/xyz.jsx')).toEqual(false);
expect(matcher('folder/xyz.tsx')).toEqual(false);
//
expect(matcher('xyz.test.js')).toEqual(true);
expect(matcher('xyz.test.ts')).toEqual(true);
expect(matcher('xyz.test.jsx')).toEqual(true);
expect(matcher('xyz.test.tsx')).toEqual(true);
expect(matcher('folder/xyz.test.js')).toEqual(true);
expect(matcher('folder/xyz.test.ts')).toEqual(true);
expect(matcher('folder/xyz.test.jsx')).toEqual(true);
expect(matcher('folder/xyz.test.tsx')).toEqual(true);
expect(matcher('folder/subfolder/xyz.test.js')).toEqual(true);
expect(matcher('folder/subfolder/xyz.test.ts')).toEqual(true);
expect(matcher('folder/subfolder/xyz.test.jsx')).toEqual(true);
expect(matcher('folder/subfolder/xyz.test.tsx')).toEqual(true);
//
expect(matcher('__tests__/subfolder/xyz.js')).toEqual(true);
expect(matcher('__tests__/subfolder/xyz.ts')).toEqual(true);
expect(matcher('__tests__/subfolder/xyz.jsx')).toEqual(true);
expect(matcher('__tests__/subfolder/xyz.tsx')).toEqual(true);
expect(matcher('folder/__tests__/xyz.js')).toEqual(true);
expect(matcher('folder/__tests__/xyz.ts')).toEqual(true);
expect(matcher('folder/__tests__/xyz.jsx')).toEqual(true);
expect(matcher('folder/__tests__/xyz.tsx')).toEqual(true);
});
});

describe('createAbsoluteFilePathMatcher', () => {
const rootFolders = ['/_root/docs', '/root/_docs/', '/__test__/website/src'];

const matcher = createAbsoluteFilePathMatcher(
GlobExcludeDefault,
rootFolders,
);

test('match default exclude MD/MDX partials correctly', () => {
expect(matcher('/_root/docs/myDoc.md')).toEqual(false);
expect(matcher('/_root/docs/myDoc.mdx')).toEqual(false);
expect(matcher('/root/_docs/myDoc.md')).toEqual(false);
expect(matcher('/root/_docs/myDoc.mdx')).toEqual(false);
expect(matcher('/_root/docs/category/myDoc.md')).toEqual(false);
expect(matcher('/_root/docs/category/myDoc.mdx')).toEqual(false);
expect(matcher('/root/_docs/category/myDoc.md')).toEqual(false);
expect(matcher('/root/_docs/category/myDoc.mdx')).toEqual(false);
//
expect(matcher('/_root/docs/_myDoc.md')).toEqual(true);
expect(matcher('/_root/docs/_myDoc.mdx')).toEqual(true);
expect(matcher('/root/_docs/_myDoc.md')).toEqual(true);
expect(matcher('/root/_docs/_myDoc.mdx')).toEqual(true);
expect(matcher('/_root/docs/_category/myDoc.md')).toEqual(true);
expect(matcher('/_root/docs/_category/myDoc.mdx')).toEqual(true);
expect(matcher('/root/_docs/_category/myDoc.md')).toEqual(true);
expect(matcher('/root/_docs/_category/myDoc.mdx')).toEqual(true);
});

test('match default exclude tests correctly', () => {
expect(matcher('/__test__/website/src/xyz.js')).toEqual(false);
expect(matcher('/__test__/website/src/__test__/xyz.js')).toEqual(true);
expect(matcher('/__test__/website/src/xyz.test.js')).toEqual(true);
});

test('throw if file is not contained in any root doc', () => {
expect(() =>
matcher('/bad/path/myDoc.md'),
).toThrowErrorMatchingInlineSnapshot(
`"createAbsoluteFilePathMatcher unexpected error, absoluteFilePath=/bad/path/myDoc.md was not contained in any of the root folders [\\"/_root/docs\\",\\"/root/_docs/\\",\\"/__test__/website/src\\"]"`,
);
});
});
30 changes: 29 additions & 1 deletion packages/docusaurus-utils/src/globUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
// Globby/Micromatch are the 2 libs we use in Docusaurus consistently

export {default as Globby} from 'globby';

import Micromatch from 'micromatch'; // Note: Micromatch is used by Globby
import path from 'path';

// The default patterns we ignore when globbing
// using _ prefix for exclusion by convention
Expand All @@ -33,3 +33,31 @@ export function createMatcher(patterns: string[]): Matcher {
);
return (str) => regexp.test(str);
}

// We use match patterns like '**/_*/**',
// This function permits to help to:
// Match /user/sebastien/website/docs/_partials/xyz.md
// Ignore /user/_sebastien/website/docs/partials/xyz.md
export function createAbsoluteFilePathMatcher(
patterns: string[],
rootFolders: string[],
): Matcher {
const matcher = createMatcher(patterns);

function getRelativeFilePath(absoluteFilePath: string) {
const rootFolder = rootFolders.find((folderPath) =>
absoluteFilePath.startsWith(folderPath),
);
if (!rootFolder) {
throw new Error(
`createAbsoluteFilePathMatcher unexpected error, absoluteFilePath=${absoluteFilePath} was not contained in any of the root folders ${JSON.stringify(
rootFolders,
)}`,
);
}
return path.relative(rootFolder, absoluteFilePath);
}

return (absoluteFilePath: string) =>
matcher(getRelativeFilePath(absoluteFilePath));
}
7 changes: 6 additions & 1 deletion packages/docusaurus-utils/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ export * from './markdownParser';
export * from './markdownLinks';
export * from './escapePath';
export {md5Hash, simpleHash, docuHash} from './hashUtils';
export {Globby, GlobExcludeDefault, createMatcher} from './globUtils';
export {
Globby,
GlobExcludeDefault,
createMatcher,
createAbsoluteFilePathMatcher,
} from './globUtils';

const fileHash = new Map();
export async function generate(
Expand Down
13 changes: 13 additions & 0 deletions website/_dogfooding/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Docusaurus website dogfooding

This is where we test edge cases of Docusaurus by using fancy configs, ensuring they all don't fail during a real site build.

Eventually, we could move these tests later so another test site? Note there is value to keep seeing the live result of those tests.

Fancy things we can test for here:

- Plugin Multi-instance
- Symlinks support
- Webpack configs
- \_ prefix convention
- Huge sidebars impact
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 3 additions & 0 deletions website/_dogfooding/_pages-tests/_pagePartial.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Page partial content

This is text coming from a page partial
9 changes: 9 additions & 0 deletions website/_dogfooding/_pages-tests/index.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
## Page

Let's import some MDX partial:

```mdx-code-block
import PagePartial from "./_pagePartial.md"
<PagePartial />
```
File renamed without changes.
File renamed without changes.
1 change: 1 addition & 0 deletions website/_dogfooding/docs-tests-symlink
Loading

0 comments on commit 4d06f26

Please sign in to comment.