Skip to content

Commit

Permalink
fix(settings): set the concurrency option to count of CPUs
Browse files Browse the repository at this point in the history
Infinity does not speed up, but takes CPU time on very large directories with a large number of "branches".
  • Loading branch information
mrmlnc committed Jun 16, 2019
1 parent d777111 commit 03201ed
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 8 deletions.
5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,11 +209,11 @@ See [Options](#options-1) section.
#### concurrency

* Type: `number`
* Default: `Infinity`
* Default: `os.cpus().length`

Specifies the maximum number of concurrent requests from a reader to read directories.

> :book: The higher the number, the higher the performance and load on the file system. If you want to read in quiet mode, set the value to a comfortable number or `4 * os.cpus().length` (4 is default size of [thread pool work scheduling][libuv_threadpool]).
> :book: The higher the number, the higher the performance and load on the file system. If you want to read in quiet mode, set the value to a comfortable number or `1`.
#### cwd

Expand Down Expand Up @@ -673,7 +673,6 @@ This software is released under the terms of the MIT license.
[glob_definition]: https://en.wikipedia.org/wiki/Glob_(programming)
[glob_linux_man]: http://man7.org/linux/man-pages/man3/glob.3.html
[intel_ssd]: https://ark.intel.com/content/www/us/en/ark/products/93012/intel-ssd-dc-s3520-series-240gb-2-5in-sata-6gb-s-3d1-mlc.html
[libuv_threadpool]: http://docs.libuv.org/en/v1.x/threadpool.html
[micromatch_backslashes]: https://github.com/micromatch/micromatch#backslashes
[micromatch_braces]: https://github.com/micromatch/braces
[micromatch_extended_globbing]: https://github.com/micromatch/micromatch#extended-globbing
Expand Down
5 changes: 3 additions & 2 deletions src/providers/provider.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,14 @@ describe('Providers → Provider', () => {

describe('.getReaderOptions', () => {
it('should return options for reader with global base (.)', () => {
const provider = getProvider();
const settings = new Settings();
const provider = getProvider(settings);
const task = tests.task.builder().base('.').positive('*').build();

const actual = provider.getReaderOptions(task);

assert.strictEqual(actual.basePath, '');
assert.strictEqual(actual.concurrency, Infinity);
assert.strictEqual(actual.concurrency, settings.concurrency);
assert.strictEqual(typeof actual.deepFilter, 'function');
assert.strictEqual(typeof actual.entryFilter, 'function');
assert.strictEqual(typeof actual.errorFilter, 'function');
Expand Down
3 changes: 2 additions & 1 deletion src/settings.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as assert from 'assert';
import * as os from 'os';

import Settings, { DEFAULT_FILE_SYSTEM_ADAPTER } from './settings';

Expand All @@ -25,7 +26,7 @@ describe('Settings', () => {
assert.ok(settings.globstar);
assert.ok(settings.onlyFiles);
assert.ok(settings.unique);
assert.strictEqual(settings.concurrency, Infinity);
assert.strictEqual(settings.concurrency, os.cpus().length);
assert.strictEqual(settings.cwd, process.cwd());
});

Expand Down
7 changes: 5 additions & 2 deletions src/settings.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import * as fs from 'fs';
import * as os from 'os';

import { FileSystemAdapter, Pattern } from './types/index';

const CPU_COUNT = os.cpus().length;

export const DEFAULT_FILE_SYSTEM_ADAPTER: FileSystemAdapter = {
lstat: fs.lstat,
lstatSync: fs.lstatSync,
Expand Down Expand Up @@ -42,7 +45,7 @@ export interface Options {
* Specifies the maximum number of concurrent requests from a reader to read
* directories.
*
* @default Infinity
* @default os.cpus().length
*/
concurrency?: number;
/**
Expand Down Expand Up @@ -154,7 +157,7 @@ export default class Settings {
public readonly baseNameMatch: boolean = this._getValue(this._options.baseNameMatch, false);
public readonly braceExpansion: boolean = this._getValue(this._options.braceExpansion, true);
public readonly caseSensitiveMatch: boolean = this._getValue(this._options.caseSensitiveMatch, true);
public readonly concurrency: number = this._getValue(this._options.concurrency, Infinity);
public readonly concurrency: number = this._getValue(this._options.concurrency, CPU_COUNT);
public readonly cwd: string = this._getValue(this._options.cwd, process.cwd());
public readonly deep: number = this._getValue(this._options.deep, Infinity);
public readonly dot: boolean = this._getValue(this._options.dot, false);
Expand Down

0 comments on commit 03201ed

Please sign in to comment.