❎ Exclude files listed in project config from myst build and watch

.changeset/wet-crabs-taste.md

@@ -0,0 +1,6 @@
+---
+'myst-config': patch
+'myst-cli': patch
+---
+
+Exclude files listed in project confg from myst build and watch

docs/table-of-contents.md

@@ -57,6 +57,27 @@ The “root” of a site is the page displayed when someone browses to the index
 All of these can be over-ridden by choosing an explicit `_toc.yml`, when that is present it will be used.
 ```
 
+### Excluding Files
+
+If there are markdown or notebook files within a project folder that you do not want included in your project, you may list these in the `myst.yml` project frontmatter under `exclude`. For example, to ignore a single file `notes.md`, all notebooks in the folder `hpc/`, and all files named `ignore.md`:
+
+```yaml
+project:
+  exclude:
+    - notes.md
+    - hpc/*.ipynb
+    - '**/ignore.md'
+```
+
+Additionally, files excluded in this way will also not be watched during `myst start`. This may be useful if you have a folder with many thousands of files that causes the `myst start` watch task to crash. For example, in the `data/` directory, there may be no markdown and no notebooks but 100,000 small data files:
+
+```yaml
+project:
+  exclude: data/**
+```
+
+Note that when these files are excluded, they can still be specifically referenced by other files in your project (e.g. in an {myst:directive}`include directives <include>` or as a download), however, a change in those files will not trigger a build. An alternative in this case is to generate a table of contents (see [](./table-of-contents.md)). By default hidden folders (those starting with `.`, like `.git`), `_build` and `node_modules` are excluded.
+
 (toc-format)=
 
 ## Defining a `_toc.yml` using Jupyter Book’s format

packages/myst-cli/src/build/site/watch.ts

@@ -28,39 +28,49 @@ function watchConfigAndPublic(session: ISession, serverReload: () => void, opts:
       ignoreInitial: true,
       awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 },
     })
-    .on('all', watchProcessor('processSite', session, null, serverReload, opts));
+    .on('all', watchProcessor(session, null, serverReload, opts));
 }
 
-function triggerProjectReload(file: string, eventType: string) {
-  // Reload project if toc changes
-  if (basename(file) === '_toc.yml') return true;
+function triggerProjectReload(
+  session: ISession,
+  file: string,
+  eventType: string,
+  projectPath?: string,
+) {
+  // Reload project if project config or toc changes
+  const state = session.store.getState();
+  const projectConfigFile = projectPath
+    ? selectors.selectLocalConfigFile(state, projectPath)
+    : selectors.selectCurrentProjectFile(state);
+  if (file === projectConfigFile || basename(file) === '_toc.yml') return true;
   // Reload project if file is added or remvoed
   if (['add', 'unlink'].includes(eventType)) return true;
   // Otherwise do not reload project
   return false;
 }
 
-async function siteProcessor(session: ISession, serverReload: () => void, opts: TransformOptions) {
-  session.log.info('💥 Triggered full site rebuild');
-  await processSite(session, opts);
-  serverReload();
-}
-
-async function fileProcessor(
+async function processorFn(
   session: ISession,
-  file: string,
+  file: string | null,
   eventType: string,
-  siteProject: SiteProject,
+  siteProject: SiteProject | null,
   serverReload: () => void,
   opts: TransformOptions,
 ) {
-  changeFile(session, file, eventType);
-  if (KNOWN_FAST_BUILDS.has(extname(file)) && eventType === 'unlink') {
-    session.log.info(`🚮 File ${file} deleted...`);
+  if (file) {
+    changeFile(session, file, eventType);
+    if (KNOWN_FAST_BUILDS.has(extname(file)) && eventType === 'unlink') {
+      session.log.info(`🚮 File ${file} deleted...`);
+    }
   }
-  if (!KNOWN_FAST_BUILDS.has(extname(file)) || ['add', 'unlink'].includes(eventType)) {
+  if (
+    !siteProject ||
+    !file ||
+    !KNOWN_FAST_BUILDS.has(extname(file)) ||
+    ['add', 'unlink'].includes(eventType)
+  ) {
     let reloadProject = false;
-    if (triggerProjectReload(file, eventType)) {
+    if (file && triggerProjectReload(session, file, eventType, siteProject?.path)) {
       session.log.info('💥 Triggered full project load and site rebuild');
       reloadProject = true;
     } else {
@@ -114,7 +124,6 @@ async function fileProcessor(
 }
 
 function watchProcessor(
-  operation: 'processSite' | 'processFile',
   session: ISession,
   siteProject: SiteProject | null,
   serverReload: () => void,
@@ -132,15 +141,14 @@ function watchProcessor(
     }
     session.store.dispatch(watch.actions.markReloading(true));
     session.log.debug(`File modified: "${file}" (${eventType})`);
-    if (operation === 'processSite' || !siteProject) {
-      await siteProcessor(session, serverReload, opts);
-    } else {
-      await fileProcessor(session, file, eventType, siteProject, serverReload, opts);
-    }
+    await processorFn(session, file, eventType, siteProject, serverReload, opts);
     while (selectors.selectReloadingState(session.store.getState()).reloadRequested) {
       // If reload(s) were requested during previous build, just reload everything once.
       session.store.dispatch(watch.actions.markReloadRequested(false));
-      await siteProcessor(session, serverReload, { reloadProject: true, ...opts });
+      await processorFn(session, null, eventType, null, serverReload, {
+        reloadProject: true,
+        ...opts,
+      });
     }
     session.store.dispatch(watch.actions.markReloading(false));
   };
@@ -164,14 +172,16 @@ export function watchContent(session: ISession, serverReload: () => void, opts:
         ? localProjects.filter(({ path }) => path !== '.').map(({ path }) => join(path, '*'))
         : [];
     if (siteConfigFile) ignored.push(siteConfigFile);
+    const projectConfig = selectors.selectLocalProjectConfig(state, proj.path);
+    if (projectConfig?.exclude) ignored.push(...projectConfig.exclude);
     const dependencies = new Set(selectors.selectAllDependencies(state, proj.path));
     chokidar
       .watch([proj.path, ...dependencies], {
         ignoreInitial: true,
         ignored: ['public', '**/_build/**', '**/.git/**', ...ignored],
         awaitWriteFinish: { stabilityThreshold: 100, pollInterval: 50 },
       })
-      .on('all', watchProcessor('processFile', session, proj, serverReload, opts));
+      .on('all', watchProcessor(session, proj, serverReload, opts));
   });
   // Watch the myst.yml
   watchConfigAndPublic(session, serverReload, opts);

packages/myst-cli/src/config.ts

@@ -140,11 +140,16 @@ export function loadConfig(session: ISession, path: string) {
   return conf;
 }
 
-export function resolveToAbsolute(session: ISession, basePath: string, relativePath: string) {
+export function resolveToAbsolute(
+  session: ISession,
+  basePath: string,
+  relativePath: string,
+  checkExists = true,
+) {
   let message: string;
   try {
     const absPath = resolve(join(basePath, relativePath));
-    if (fs.existsSync(absPath)) {
+    if (!checkExists || fs.existsSync(absPath)) {
       return absPath;
     }
     message = `Does not exist as local path: ${absPath}`;
@@ -155,10 +160,15 @@ export function resolveToAbsolute(session: ISession, basePath: string, relativeP
   return relativePath;
 }
 
-function resolveToRelative(session: ISession, basePath: string, absPath: string) {
+function resolveToRelative(
+  session: ISession,
+  basePath: string,
+  absPath: string,
+  checkExists = true,
+) {
   let message: string;
   try {
-    if (fs.existsSync(absPath)) {
+    if (!checkExists || fs.existsSync(absPath)) {
       // If it is the same path, use a '.'
       return relative(basePath, absPath) || '.';
     }
@@ -174,7 +184,12 @@ function resolveSiteConfigPaths(
   session: ISession,
   path: string,
   siteConfig: SiteConfig,
-  resolutionFn: (session: ISession, basePath: string, path: string) => string,
+  resolutionFn: (
+    session: ISession,
+    basePath: string,
+    path: string,
+    checkExists?: boolean,
+  ) => string,
 ) {
   const resolvedFields: SiteConfig = {};
   if (siteConfig.projects) {
@@ -195,7 +210,12 @@ function resolveProjectConfigPaths(
   session: ISession,
   path: string,
   projectConfig: ProjectConfig,
-  resolutionFn: (session: ISession, basePath: string, path: string) => string,
+  resolutionFn: (
+    session: ISession,
+    basePath: string,
+    path: string,
+    checkExists?: boolean,
+  ) => string,
 ) {
   const resolvedFields: ProjectConfig = {};
   if (projectConfig.bibliography) {
@@ -208,7 +228,7 @@ function resolveProjectConfigPaths(
   }
   if (projectConfig.exclude) {
     resolvedFields.exclude = projectConfig.exclude.map((file) => {
-      return resolutionFn(session, path, file);
+      return resolutionFn(session, path, file, false);
     });
   }
   if (projectConfig.plugins) {

packages/myst-cli/src/project/fromPath.ts

@@ -1,8 +1,10 @@
 import fs from 'node:fs';
-import { extname, join } from 'node:path';
+import { extname, join, sep } from 'node:path';
+import { glob } from 'glob';
 import { isDirectory } from 'myst-cli-utils';
 import { RuleId } from 'myst-common';
 import type { ISession } from '../session/types.js';
+import { selectors } from '../store/index.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
 import { fileInfo } from '../utils/fileInfo.js';
 import { nextLevel } from '../utils/nextLevel.js';
@@ -136,36 +138,51 @@ function indexFileFromPages(pages: (LocalProjectFolder | LocalProjectPage)[], pa
 /**
  * Build project structure from local file/folder structure.
  */
-export function projectFromPath(
+export async function projectFromPath(
   session: ISession,
   path: string,
   indexFile?: string,
-): Omit<LocalProject, 'bibliography'> {
+): Promise<Omit<LocalProject, 'bibliography'>> {
   const ext_string = VALID_FILE_EXTENSIONS.join(' or ');
   if (indexFile) {
     if (!isValidFile(indexFile))
       throw Error(`Index file ${indexFile} has invalid extension; must be ${ext_string}}`);
     if (!fs.existsSync(indexFile)) throw Error(`Index file ${indexFile} not found`);
   }
   const rootConfigYamls = session.configFiles.map((file) => join(path, file));
+  const projectConfig = selectors.selectLocalProjectConfig(session.store.getState(), path);
+  const excludePatterns = projectConfig?.exclude ?? [];
+  const excludeFiles = (
+    await Promise.all(
+      excludePatterns.map(async (pattern) => {
+        const matches = await glob(pattern.split(sep).join('/'));
+        return matches
+          .map((match) => match.split('/').join(sep))
+          .filter((match) => isValidFile(match));
+      }),
+    )
+  ).flat();
+  const ignoreFiles = [...rootConfigYamls, ...excludeFiles];
+  let implicitIndex = false;
   if (!indexFile) {
     const searchPages = projectPagesFromPath(
       session,
       path,
       1,
       {},
-      { ignore: rootConfigYamls, suppressWarnings: true },
+      { ignore: ignoreFiles, suppressWarnings: true },
     );
     if (!searchPages.length) {
       throw Error(`No valid files with extensions ${ext_string} found in path "${path}"`);
     }
     indexFile = indexFileFromPages(searchPages, path);
     if (!indexFile) throw Error(`Unable to find any index file in path "${path}"`);
+    implicitIndex = true;
   }
   const pageSlugs: PageSlugs = {};
   const { slug } = fileInfo(indexFile, pageSlugs);
   const pages = projectPagesFromPath(session, path, 1, pageSlugs, {
-    ignore: [indexFile, ...rootConfigYamls],
+    ignore: [indexFile, ...ignoreFiles],
   });
-  return { file: indexFile, index: slug, path, pages };
+  return { file: indexFile, index: slug, path, pages, implicitIndex };
 }

packages/myst-cli/src/project/load.ts

@@ -58,10 +58,11 @@ export async function loadProjectFromDisk(
     writeToc = false;
   } else {
     const project = selectors.selectLocalProject(session.store.getState(), path);
-    if (!index && project?.file) {
+    if (!index && !project?.implicitIndex && project?.file) {
+      // If there is no new index, keep the original unless it was implicit previously
       index = project.file;
     }
-    newProject = projectFromPath(session, path, index);
+    newProject = await projectFromPath(session, path, index);
   }
   if (!newProject) {
     throw new Error(`Could not load project from ${path}`);