Skip to content

Commit

Permalink
[file-search] spawn rigrep with root URI as cwd and only once for eac…
Browse files Browse the repository at this point in the history 
…h root

Signed-off-by: Anton Kosyakov <[email protected]>
  • Loading branch information
@akosyakov
akosyakov committed Mar 15, 2019
1 parent 79ded6c commit b21023e
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 157 deletions.
1 change: 1 addition & 0 deletions packages/file-search/src/common/file-search-service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,6 @@ export namespace FileSearchService {
useGitIgnore?: boolean
/** when `undefined`, no excludes will apply, when empty array, default excludes will apply */
defaultIgnorePatterns?: string[]
includePatterns?: string[]
}
}
50 changes: 24 additions & 26 deletions packages/file-search/src/node/file-search-service-impl.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,14 @@ describe('search-service', function () {
expect(testFile).to.be.not.undefined;
});

it('shall respect nested .gitignore');
// const service = testContainer.get(FileSearchServiceImpl);
// const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources')).toString();
// const matches = await service.find('foo', { rootUri, fuzzyMatch: false });
it.skip('shall respect nested .gitignore', async () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources')).toString();
const matches = await service.find('foo', { rootUris: [rootUri], fuzzyMatch: false });

// expect(matches.find(match => match.endsWith('subdir1/sub-bar/foo.txt'))).to.be.undefined;
// expect(matches.find(match => match.endsWith('subdir1/sub2/foo.txt'))).to.be.not.undefined;
// expect(matches.find(match => match.endsWith('subdir1/foo.txt'))).to.be.not.undefined;
// });
expect(matches.find(match => match.endsWith('subdir1/sub-bar/foo.txt'))).to.be.undefined;
expect(matches.find(match => match.endsWith('subdir1/sub2/foo.txt'))).to.be.not.undefined;
expect(matches.find(match => match.endsWith('subdir1/foo.txt'))).to.be.not.undefined;
});

it('shall cancel searches', async () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../../../..')).toString();
Expand All @@ -85,19 +84,19 @@ describe('search-service', function () {
it('should support file searches with globs', async () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources/subdir1/sub2')).toString();

const matches = await service.find('**/*oo.*', { rootUris: [rootUri] });
const matches = await service.find('', { rootUris: [rootUri], includePatterns: ['**/*oo.*'] });
expect(matches).to.be.not.undefined;
expect(matches.length).to.eq(1);
});

it('should support file searches with globs without the prefixed or trailing star (*)', async () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources/subdir1/sub2')).toString();

const trailingMatches = await service.find('*oo', { rootUris: [rootUri] });
const trailingMatches = await service.find('', { rootUris: [rootUri], includePatterns: ['*oo'] });
expect(trailingMatches).to.be.not.undefined;
expect(trailingMatches.length).to.eq(1);

const prefixedMatches = await service.find('oo*', { rootUris: [rootUri] });
const prefixedMatches = await service.find('', { rootUris: [rootUri], includePatterns: ['oo*'] });
expect(prefixedMatches).to.be.not.undefined;
expect(prefixedMatches.length).to.eq(1);
});
Expand All @@ -107,45 +106,44 @@ describe('search-service', function () {
it('should ignore strings passed through the search options', async () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources/subdir1/sub2')).toString();

const matches = await service.find('**/*oo.*', { rootUris: [rootUri], defaultIgnorePatterns: ['foo'] });
const matches = await service.find('', { rootUris: [rootUri], includePatterns: ['**/*oo.*'], defaultIgnorePatterns: ['foo'] });
expect(matches).to.be.not.undefined;
expect(matches.length).to.eq(0);
});

it('should ignore globs passed through the search options', async () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../test-resources/subdir1/sub2')).toString();

const matches = await service.find('**/*oo.*', { rootUris: [rootUri], defaultIgnorePatterns: ['*fo*'] });
const matches = await service.find('', { rootUris: [rootUri], includePatterns: ['**/*oo.*'], defaultIgnorePatterns: ['*fo*'] });
expect(matches).to.be.not.undefined;
expect(matches.length).to.eq(0);
});
});

describe('irrelevant absolute results', () => {
const rootUri = FileUri.create(path.resolve(__dirname, '../../../../..'));
const rootUri = FileUri.create(path.resolve(__dirname, '../../../..'));

it('not fuzzy', async () => {
const matches = await service.find('theia', { rootUris: [rootUri.toString()], fuzzyMatch: false, useGitIgnore: true, limit: 200 });
const searchPattern = rootUri.path.dir.base;
const matches = await service.find(searchPattern, { rootUris: [rootUri.toString()], fuzzyMatch: false, useGitIgnore: true, limit: 200 });
for (const match of matches) {
const relativUri = rootUri.relative(new URI(match));
if (relativUri) {
const relativMatch = relativUri.toString();
assert.notEqual(relativMatch.indexOf('theia'), -1, relativMatch);
}
assert.notEqual(relativUri, undefined);
const relativMatch = relativUri!.toString();
assert.notEqual(relativMatch.indexOf(searchPattern), -1, relativMatch);
}
});

it('fuzzy', async () => {
const matches = await service.find('shell', { rootUris: [rootUri.toString()], fuzzyMatch: true, useGitIgnore: true, limit: 200 });
for (const match of matches) {
const relativUri = rootUri.relative(new URI(match));
if (relativUri) {
const relativMatch = relativUri.toString();
let position = 0;
for (const ch of 'shell') {
position = relativMatch.indexOf(ch, position);
assert.notEqual(position, -1, relativMatch);
}
assert.notEqual(relativUri, undefined);
const relativMatch = relativUri!.toString();
let position = 0;
for (const ch of 'shell') {
position = relativMatch.indexOf(ch, position);
assert.notEqual(position, -1, relativMatch);
}
}
});
Expand Down
197 changes: 74 additions & 123 deletions packages/file-search/src/node/file-search-service-impl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,15 @@
* SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
********************************************************************************/

import * as readline from 'readline';
import * as fuzzy from 'fuzzy';
import * as readline from 'readline';
import { rgPath } from 'vscode-ripgrep';
import { injectable, inject } from 'inversify';
import URI from '@theia/core/lib/common/uri';
import { FileSearchService } from '../common/file-search-service';
import { RawProcessFactory } from '@theia/process/lib/node';
import { rgPath } from 'vscode-ripgrep';
import { Deferred } from '@theia/core/lib/common/promise-util';
import { FileUri } from '@theia/core/lib/node/file-uri';
import { CancellationToken, ILogger } from '@theia/core';
import { CancellationTokenSource, CancellationToken, ILogger } from '@theia/core';
import { RawProcessFactory } from '@theia/process/lib/node';
import { FileSearchService } from '../common/file-search-service';

@injectable()
export class FileSearchServiceImpl implements FileSearchService {
Expand All @@ -32,7 +31,13 @@ export class FileSearchServiceImpl implements FileSearchService {
@inject(ILogger) protected readonly logger: ILogger,
@inject(RawProcessFactory) protected readonly rawProcessFactory: RawProcessFactory) { }

async find(searchPattern: string, options: FileSearchService.Options, cancellationToken?: CancellationToken): Promise<string[]> {
async find(searchPattern: string, options: FileSearchService.Options, clientToken?: CancellationToken): Promise<string[]> {
const cancellationSource = new CancellationTokenSource();
if (clientToken) {
clientToken.onCancellationRequested(() => cancellationSource.cancel());
}
const token = cancellationSource.token;

if (options.defaultIgnorePatterns && options.defaultIgnorePatterns.length === 0) { // default excludes
options.defaultIgnorePatterns.push('.git');
}
Expand All @@ -42,122 +47,79 @@ export class FileSearchServiceImpl implements FileSearchService {
useGitIgnore: true,
...options
};
const args: string[] = this.getSearchArgs(opts);

const resultDeferred = new Deferred<string[]>();
try {
const results = await Promise.all([
this.doGlobSearch(searchPattern, args.slice(), opts.limit, cancellationToken),
this.doStringSearch(searchPattern, args.slice(), opts, cancellationToken)
]);
const matches = Array.from(new Set([...results[0], ...results[1].exactMatches])).sort().slice(0, opts.limit);
if (matches.length === opts.limit) {
resultDeferred.resolve(matches);
} else {
resultDeferred.resolve([...matches, ...results[1].fuzzyMatches].slice(0, opts.limit));
const exactMatches = new Set<string>();
const fuzzyMatches = new Set<string>();
const stringPattern = searchPattern.toLocaleLowerCase();
await Promise.all(options.rootUris.map(async root => {
try {
const rootUri = new URI(root);
await this.doFind(rootUri, searchPattern, opts, candidate => {
const fileUri = rootUri.resolve(candidate).toString();
if (exactMatches.has(fileUri) || fuzzyMatches.has(fileUri)) {
return;
}
if (!searchPattern || searchPattern === '*' || candidate.toLocaleLowerCase().indexOf(stringPattern) !== -1) {
exactMatches.add(fileUri);
} else if (opts.fuzzyMatch && fuzzy.test(searchPattern, candidate)) {
fuzzyMatches.add(fileUri);
}
if (exactMatches.size + fuzzyMatches.size === opts.limit) {
cancellationSource.cancel();
}
}, token);
} catch (e) {
console.error('Failed to search:', root, e);
}
} catch (e) {
resultDeferred.reject(e);
}));
if (clientToken && clientToken.isCancellationRequested) {
return [];
}
return resultDeferred.promise;
return [...exactMatches, ...fuzzyMatches];
}

private doGlobSearch(globPattern: string, searchArgs: string[], limit: number, cancellationToken?: CancellationToken): Promise<string[]> {
const resultDeferred = new Deferred<string[]>();
let glob = globPattern;
if (!glob.endsWith('*')) {
glob = `${glob}*`;
}
if (!glob.startsWith('*')) {
glob = `*${glob}`;
}
searchArgs.unshift(glob);
searchArgs.unshift('--glob');
const process = this.rawProcessFactory({
command: rgPath,
args: searchArgs
});
this.setupCancellation(() => {
this.logger.debug('Search cancelled');
process.kill();
resultDeferred.resolve([]);
}, cancellationToken);
const lineReader = readline.createInterface({
input: process.output,
output: process.input
});
const result: string[] = [];
lineReader.on('line', line => {
if (result.length >= limit) {
process.kill();
} else {
const fileUriStr = FileUri.create(line).toString();
result.push(fileUriStr);
}
});
process.output.on('close', () => {
resultDeferred.resolve(result);
});
process.onError(e => {
resultDeferred.reject(e);
});
return resultDeferred.promise;
}
private doFind(rootUri: URI, searchPattern: string, options: FileSearchService.Options,
accept: (fileUri: string) => void, token: CancellationToken): Promise<void> {
return new Promise((resolve, reject) => {
try {
const cwd = FileUri.fsPath(rootUri);
const args = this.getSearchArgs(options);
// TODO: why not just child_process.spawn, theia process are supposed to be used for user processes like tasks and terminals, not internal
const process = this.rawProcessFactory({ command: rgPath, args, options: { cwd } });
process.onError(reject);
process.output.on('close', resolve);
token.onCancellationRequested(() => process.kill());

private doStringSearch(
stringPattern: string, searchArgs: string[], options: Required<Pick<FileSearchService.Options, 'limit' | 'fuzzyMatch' | 'rootUris'>>, cancellationToken?: CancellationToken
): Promise<{ exactMatches: string[], fuzzyMatches: string[] }> {
const resultDeferred = new Deferred<{ exactMatches: string[], fuzzyMatches: string[] }>();
const process = this.rawProcessFactory({
command: rgPath,
args: searchArgs
});
this.setupCancellation(() => {
this.logger.debug('Search cancelled');
process.kill();
resultDeferred.resolve({ exactMatches: [], fuzzyMatches: [] });
}, cancellationToken);
const rootUris = options.rootUris.map(uri => new URI(uri));
const lineReader = readline.createInterface({
input: process.output,
output: process.input
});
const exactMatches: string[] = [];
const fuzzyMatches: string[] = [];
lineReader.on('line', line => {
if (exactMatches.length >= options.limit) {
process.kill();
} else {
const fileUri = FileUri.create(line);
for (const rootUri of rootUris) {
const relativeUri = rootUri.relative(fileUri);
if (relativeUri) {
const relativeUriStr = relativeUri.toString();
if (relativeUriStr.toLocaleLowerCase().indexOf(stringPattern.toLocaleLowerCase()) !== -1) {
exactMatches.push(fileUri.toString());
return;
} else if (options.fuzzyMatch && fuzzy.test(stringPattern, relativeUriStr)) {
fuzzyMatches.push(fileUri.toString());
return;
}
const lineReader = readline.createInterface({
input: process.output,
output: process.input
});
lineReader.on('line', line => {
if (token.isCancellationRequested) {
process.kill();
} else {
accept(line);
}
}
});
} catch (e) {
reject(e);
}
});
process.output.on('close', () => {
const fuzzyResult = fuzzyMatches.slice(0, options.limit - exactMatches.length);
resultDeferred.resolve({ exactMatches, fuzzyMatches: fuzzyResult });
});
process.onError(e => {
resultDeferred.reject(e);
});
return resultDeferred.promise;
}

private getSearchArgs(options: FileSearchService.Options): string[] {
const args: string[] = [
'--files'
];
const args = ['--files', '--case-sensitive'];
if (options.includePatterns) {
for (const includePattern of options.includePatterns) {
let includeGlob = includePattern;
if (!includeGlob.endsWith('*')) {
includeGlob = `${includeGlob}*`;
}
if (!includeGlob.startsWith('*')) {
includeGlob = `*${includeGlob}`;
}
args.push('--glob', includeGlob);
}
}
if (!options.useGitIgnore) {
args.push('-uu');
}
Expand All @@ -176,18 +138,7 @@ export class FileSearchServiceImpl implements FileSearchService {
args.push(ignore);
});
}
args.push(...options.rootUris.map(r => FileUri.fsPath(r)));
return args;
}

private setupCancellation(onCancel: () => void, cancellationToken?: CancellationToken) {
if (cancellationToken) {
if (cancellationToken.isCancellationRequested) {
onCancel();
} else {
cancellationToken.onCancellationRequested(onCancel);
}
}
}

}
2 changes: 1 addition & 1 deletion packages/plugin-ext/src/api/plugin-api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ export interface QuickOpenMain {

export interface WorkspaceMain {
$pickWorkspaceFolder(options: WorkspaceFolderPickOptionsMain): Promise<theia.WorkspaceFolder | undefined>;
$startFileSearch(includePattern: string, excludePatternOrDisregardExcludes: string | false,
$startFileSearch(includePattern: string, includeFolder: string | undefined, excludePatternOrDisregardExcludes: string | false,
maxResults: number | undefined, token: theia.CancellationToken): PromiseLike<UriComponents[]>;
$registerTextDocumentContentProvider(scheme: string): Promise<void>;
$unregisterTextDocumentContentProvider(scheme: string): void;
Expand Down
14 changes: 8 additions & 6 deletions packages/plugin-ext/src/main/browser/workspace-main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,22 +152,24 @@ export class WorkspaceMainImpl implements WorkspaceMain {
});
}

async $startFileSearch(includePattern: string, excludePatternOrDisregardExcludes?: string | false,
maxResults?: number, token?: theia.CancellationToken): Promise<UriComponents[]> {
const opts: FileSearchService.Options = { rootUris: this.roots.map(r => r.uri) };
async $startFileSearch(includePattern: string, includeFolderUri: string | undefined, excludePatternOrDisregardExcludes?: string | false,
maxResults?: number): Promise<UriComponents[]> {
const rootUris = includeFolderUri ? [includeFolderUri] : this.roots.map(r => r.uri);
const opts: FileSearchService.Options = { rootUris };
if (includePattern) {
opts.includePatterns = [includePattern];
}
if (typeof excludePatternOrDisregardExcludes === 'string') {
if (excludePatternOrDisregardExcludes === '') { // default excludes
opts.defaultIgnorePatterns = [];
} else {
opts.defaultIgnorePatterns = [excludePatternOrDisregardExcludes];
}
} else {
opts.defaultIgnorePatterns = undefined; // no excludes
}
if (typeof maxResults === 'number') {
opts.limit = maxResults;
}
const uriStrs = await this.fileSearchService.find(includePattern, opts);
const uriStrs = await this.fileSearchService.find('', opts);
return uriStrs.map(uriStr => Uri.parse(uriStr));
}

Expand Down
Loading

0 comments on commit b21023e

Please sign in to comment.