From 9d3609e314b5e6b3e9adf3a045c4eee934c03d94 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 2 Mar 2023 13:30:10 -0800 Subject: [PATCH] add maxDepth option Fix: #296 --- README.md | 3 + src/glob.ts | 27 ++++++ src/index.ts | 60 ++++++------- src/walker.ts | 16 ++++ tap-snapshots/test/max-depth.ts.test.cjs | 38 ++++++++ test/max-depth.ts | 109 +++++++++++++++++++++++ 6 files changed, 219 insertions(+), 34 deletions(-) create mode 100644 tap-snapshots/test/max-depth.ts.test.cjs create mode 100644 test/max-depth.ts diff --git a/README.md b/README.md index 96d3459c..e67e727c 100644 --- a/README.md +++ b/README.md @@ -310,6 +310,9 @@ share the previously loaded cache. systems, or `false` on case-insensitive file systems, then the walk may return more or less results than expected. +- `maxDepth` Specify a number to limit the depth of the directory + traversal to this many levels below the `cwd`. + - `matchBase` Perform a basename-only match if the pattern does not contain any slash characters. That is, `*.js` would be treated as equivalent to `**/*.js`, matching all js files in diff --git a/src/glob.ts b/src/glob.ts index 584b156d..7b4e9b89 100644 --- a/src/glob.ts +++ b/src/glob.ts @@ -126,6 +126,14 @@ export interface GlobOptions { */ matchBase?: boolean + /** + * Limit the directory traversal to a given depth below the cwd. + * Note that this does NOT prevent traversal to sibling folders, + * root patterns, and so on. It only limits the maximum folder depth + * that the walk will descend, relative to the cwd. + */ + maxDepth?: number + /** * Do not expand `{a,b}` and `{1..3}` brace sets. */ @@ -293,6 +301,7 @@ export class Glob implements GlobOptions { magicalBraces: boolean mark?: boolean matchBase: boolean + maxDepth: number nobrace: boolean nocase: boolean nodir: boolean @@ -351,6 +360,8 @@ export class Glob implements GlobOptions { this.noglobstar = !!opts.noglobstar this.matchBase = !!opts.matchBase + this.maxDepth = + typeof opts.maxDepth === 'number' ? opts.maxDepth : Infinity if (this.withFileTypes && this.absolute !== undefined) { throw new Error('cannot set absolute and withFileTypes:true') @@ -445,6 +456,10 @@ export class Glob implements GlobOptions { return [ ...(await new GlobWalker(this.patterns, this.scurry.cwd, { ...this.opts, + maxDepth: + this.maxDepth !== Infinity + ? this.maxDepth + this.scurry.cwd.depth() + : Infinity, platform: this.platform, nocase: this.nocase, }).walk()), @@ -459,6 +474,10 @@ export class Glob implements GlobOptions { return [ ...new GlobWalker(this.patterns, this.scurry.cwd, { ...this.opts, + maxDepth: + this.maxDepth !== Infinity + ? this.maxDepth + this.scurry.cwd.depth() + : Infinity, platform: this.platform, nocase: this.nocase, }).walkSync(), @@ -472,6 +491,10 @@ export class Glob implements GlobOptions { stream(): Minipass { return new GlobStream(this.patterns, this.scurry.cwd, { ...this.opts, + maxDepth: + this.maxDepth !== Infinity + ? this.maxDepth + this.scurry.cwd.depth() + : Infinity, platform: this.platform, nocase: this.nocase, }).stream() @@ -484,6 +507,10 @@ export class Glob implements GlobOptions { streamSync(): Minipass { return new GlobStream(this.patterns, this.scurry.cwd, { ...this.opts, + maxDepth: + this.maxDepth !== Infinity + ? this.maxDepth + this.scurry.cwd.depth() + : Infinity, platform: this.platform, nocase: this.nocase, }).streamSync() diff --git a/src/index.ts b/src/index.ts index 0cd7be2a..4167d0bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,20 +1,14 @@ import { escape, unescape } from 'minimatch' +import Minipass from 'minipass' +import { Path } from 'path-scurry' import type { GlobOptions, GlobOptionsWithFileTypesFalse, GlobOptionsWithFileTypesTrue, GlobOptionsWithFileTypesUnset, - Results, } from './glob.js' import { Glob } from './glob.js' import { hasMagic } from './has-magic.js' -import type { - GWOFileTypesFalse, - GWOFileTypesTrue, - GWOFileTypesUnset, - MatchStream, - Result, -} from './walker.js' /** * Syncronous form of {@link globStream}. Will read all the matches as fast as @@ -24,19 +18,19 @@ import type { export function globStreamSync( pattern: string | string[], options: GlobOptionsWithFileTypesTrue -): MatchStream +): Minipass export function globStreamSync( pattern: string | string[], options: GlobOptionsWithFileTypesFalse -): MatchStream +): Minipass export function globStreamSync( pattern: string | string[], options: GlobOptionsWithFileTypesUnset -): MatchStream +): Minipass export function globStreamSync( pattern: string | string[], options: GlobOptions -): MatchStream +): Minipass | Minipass export function globStreamSync( pattern: string | string[], options: GlobOptions = {} @@ -51,19 +45,19 @@ export function globStreamSync( export function globStream( pattern: string | string[], options: GlobOptionsWithFileTypesFalse -): MatchStream +): Minipass export function globStream( pattern: string | string[], options: GlobOptionsWithFileTypesTrue -): MatchStream +): Minipass export function globStream( pattern: string | string[], options?: GlobOptionsWithFileTypesUnset | undefined -): MatchStream +): Minipass export function globStream( pattern: string | string[], options: GlobOptions -): MatchStream +): Minipass | Minipass export function globStream( pattern: string | string[], options: GlobOptions = {} @@ -77,19 +71,19 @@ export function globStream( export function globSync( pattern: string | string[], options: GlobOptionsWithFileTypesFalse -): Results +): string[] export function globSync( pattern: string | string[], options: GlobOptionsWithFileTypesTrue -): Results +): Path[] export function globSync( pattern: string | string[], options?: GlobOptionsWithFileTypesUnset | undefined -): Results +): string[] export function globSync( pattern: string | string[], options: GlobOptions -): Results +): Path[] | string[] export function globSync( pattern: string | string[], options: GlobOptions = {} @@ -106,19 +100,19 @@ export function globSync( export async function glob( pattern: string | string[], options?: GlobOptionsWithFileTypesUnset | undefined -): Promise> +): Promise export async function glob( pattern: string | string[], options: GlobOptionsWithFileTypesTrue -): Promise> +): Promise export async function glob( pattern: string | string[], options: GlobOptionsWithFileTypesFalse -): Promise> +): Promise export async function glob( pattern: string | string[], options: GlobOptions -): Promise> +): Promise export async function glob( pattern: string | string[], options: GlobOptions = {} @@ -132,19 +126,19 @@ export async function glob( export function globIterate( pattern: string | string[], options?: GlobOptionsWithFileTypesUnset | undefined -): AsyncGenerator, void, void> +): AsyncGenerator export function globIterate( pattern: string | string[], options: GlobOptionsWithFileTypesTrue -): AsyncGenerator, void, void> +): AsyncGenerator export function globIterate( pattern: string | string[], options: GlobOptionsWithFileTypesFalse -): AsyncGenerator, void, void> +): AsyncGenerator export function globIterate( pattern: string | string[], options: GlobOptions -): AsyncGenerator, void, void> +): AsyncGenerator | AsyncGenerator export function globIterate( pattern: string | string[], options: GlobOptions = {} @@ -158,19 +152,19 @@ export function globIterate( export function globIterateSync( pattern: string | string[], options?: GlobOptionsWithFileTypesUnset | undefined -): Generator, void, void> +): Generator export function globIterateSync( pattern: string | string[], options: GlobOptionsWithFileTypesTrue -): Generator, void, void> +): Generator export function globIterateSync( pattern: string | string[], options: GlobOptionsWithFileTypesFalse -): Generator, void, void> +): Generator export function globIterateSync( pattern: string | string[], options: GlobOptions -): Generator, void, void> +): Generator | Generator export function globIterateSync( pattern: string | string[], options: GlobOptions = {} @@ -186,8 +180,6 @@ export type { GlobOptionsWithFileTypesFalse, GlobOptionsWithFileTypesTrue, GlobOptionsWithFileTypesUnset, - Result, - Results, } from './glob.js' export { hasMagic } from './has-magic.js' export type { MatchStream } from './walker.js' diff --git a/src/walker.ts b/src/walker.ts index ba0c022c..05810776 100644 --- a/src/walker.ts +++ b/src/walker.ts @@ -26,6 +26,9 @@ export interface GlobWalkerOpts { ignore?: string | string[] | Ignore mark?: boolean matchBase?: boolean + // Note: maxDepth here means "maximum actual Path.depth()", + // not "maximum depth beyond cwd" + maxDepth?: number nobrace?: boolean nocase?: boolean nodir?: boolean @@ -100,6 +103,7 @@ export abstract class GlobUtil { #ignore?: Ignore #sep: '\\' | '/' signal?: AbortSignal + maxDepth: number constructor(patterns: Pattern[], path: Path, opts: O) constructor(patterns: Pattern[], path: Path, opts: O) { @@ -110,6 +114,11 @@ export abstract class GlobUtil { if (opts.ignore) { this.#ignore = makeIgnore(opts.ignore, opts) } + // ignore, always set with maxDepth, but it's optional on the + // GlobOptions type + /* c8 ignore start */ + this.maxDepth = opts.maxDepth || Infinity + /* c8 ignore stop */ if (opts.signal) { this.signal = opts.signal this.signal.addEventListener('abort', () => { @@ -166,6 +175,7 @@ export abstract class GlobUtil { matchCheckTest(e: Path | undefined, ifDir: boolean): Path | undefined { return e && + (this.maxDepth === Infinity || e.depth() <= this.maxDepth) && !this.#ignored(e) && (!ifDir || e.canReaddir()) && (!this.opts.nodir || !e.isDirectory()) @@ -255,6 +265,9 @@ export abstract class GlobUtil { } for (const t of processor.subwalkTargets()) { + if (this.maxDepth !== Infinity && t.depth() >= this.maxDepth) { + continue + } tasks++ const childrenCached = t.readdirCached() if (t.calledReaddir()) @@ -333,6 +346,9 @@ export abstract class GlobUtil { } for (const t of processor.subwalkTargets()) { + if (this.maxDepth !== Infinity && t.depth() >= this.maxDepth) { + continue + } tasks++ const children = t.readdirSync() this.walkCB3Sync(t, children, processor, next) diff --git a/tap-snapshots/test/max-depth.ts.test.cjs b/tap-snapshots/test/max-depth.ts.test.cjs new file mode 100644 index 00000000..55860f2b --- /dev/null +++ b/tap-snapshots/test/max-depth.ts.test.cjs @@ -0,0 +1,38 @@ +/* IMPORTANT + * This snapshot file is auto-generated, but designed for humans. + * It should be checked into source control and tracked carefully. + * Re-generate by setting TAP_SNAPSHOT=1 and running tests. + * Make sure to inspect the output below. Do not ignore changes! + */ +'use strict' +exports[`test/max-depth.ts TAP set maxDepth > async results 1`] = ` +Array [ + "", + "a", + "a/abcdef", + "a/abcfed", + "a/b", + "a/bc", + "a/c", + "a/cb", + "a/symlink", + "a/x", + "a/z", +] +` + +exports[`test/max-depth.ts TAP set maxDepth > sync results 1`] = ` +Array [ + "", + "a", + "a/abcdef", + "a/abcfed", + "a/b", + "a/bc", + "a/c", + "a/cb", + "a/symlink", + "a/x", + "a/z", +] +` diff --git a/test/max-depth.ts b/test/max-depth.ts new file mode 100644 index 00000000..3f2c68a8 --- /dev/null +++ b/test/max-depth.ts @@ -0,0 +1,109 @@ +import { resolve } from 'path' +import { PathScurry } from 'path-scurry' +import t from 'tap' +import { Glob, glob, globStream, globStreamSync, globSync } from '../' +const j = (a: string[]) => + a + .map(s => s.replace(/\\/g, '/')) + .sort((a, b) => a.localeCompare(b, 'en')) +t.test('set maxDepth', async t => { + const maxDepth = 2 + const cwd = resolve(__dirname, 'fixtures') + const startDepth = new PathScurry(cwd).cwd.depth() + const pattern = '{*/*/*/**,*/*/**,**}' + const asyncRes = await glob(pattern, { + cwd, + maxDepth, + follow: true, + withFileTypes: true, + }) + const syncRes = globSync(pattern, { + cwd, + maxDepth, + follow: true, + withFileTypes: true, + }) + const noMaxDepth = globSync(pattern, { + cwd, + follow: true, + withFileTypes: true, + }) + const expect = j( + noMaxDepth + .filter(p => p.depth() <= startDepth + maxDepth) + .map(p => p.relative()) + ) + + const ssync = j(syncRes.map(p => p.relative())) + const sasync = j(asyncRes.map(p => p.relative())) + t.matchSnapshot(sasync, 'async results') + t.matchSnapshot(ssync, 'sync results') + t.same(ssync, expect, 'got all results sync') + t.same(sasync, expect, 'got all results async') + for (const p of syncRes) { + t.ok(p.depth() <= startDepth + maxDepth, 'does not exceed maxDepth', { + max: startDepth + maxDepth, + actual: p.depth(), + file: p.relative(), + results: 'sync', + }) + } + for (const p of asyncRes) { + t.ok(p.depth() <= startDepth + maxDepth, 'does not exceed maxDepth', { + max: startDepth + maxDepth, + actual: p.depth(), + file: p.relative(), + results: 'async', + }) + } + + t.same( + j( + await globStream(pattern, { cwd, maxDepth, follow: true }).collect() + ), + expect, + 'maxDepth with stream' + ) + t.same( + j( + await globStreamSync(pattern, { + cwd, + maxDepth, + follow: true, + }).collect() + ), + expect, + 'maxDepth with streamSync' + ) + + t.same( + await glob(pattern, { cwd, maxDepth: -1, follow: true }), + [], + 'async maxDepth -1' + ) + t.same( + globSync(pattern, { cwd, maxDepth: -1, follow: true }), + [], + 'sync maxDepth -1' + ) + + t.same( + await glob(pattern, { cwd, maxDepth: 0, follow: true }), + [''], + 'async maxDepth 0' + ) + t.same( + globSync(pattern, { cwd, maxDepth: 0, follow: true }), + [''], + 'async maxDepth 0' + ) + + const g = new Glob(pattern, { cwd, follow: true, maxDepth }) + t.same(j([...g]), expect, 'maxDepth with iteration') + const ai = new Glob(pattern, g) + const aires: string[] = [] + for await (const res of ai) { + aires.push(res) + } + t.same(j(aires), expect, 'maxDepth with async iteration') +})