diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index f253461d9..234824d77 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -24,7 +24,7 @@ import { } from './file.js'; import pLimit from 'p-limit'; import * as path from 'path'; -import {createReadStream, promises as fsp} from 'fs'; +import {createReadStream, existsSync, promises as fsp} from 'fs'; import {CRC32C} from './crc32c.js'; import {GoogleAuth} from 'google-auth-library'; import {XMLParser, XMLBuilder} from 'fast-xml-parser'; @@ -108,6 +108,7 @@ export interface DownloadManyFilesOptions { prefix?: string; stripPrefix?: string; passthroughOptions?: DownloadOptions; + skipIfExists?: boolean; } export interface DownloadFileInChunksOptions { @@ -524,6 +525,8 @@ export class TransferManager { * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files. * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through * to each individual download operation. + * @property {boolean} [skipIfExists] Do not download the file if it already exists in + * the destination. * */ /** @@ -605,6 +608,12 @@ export class TransferManager { if (options.stripPrefix) { passThroughOptionsCopy.destination = file.name.replace(regex, ''); } + if ( + options.skipIfExists && + existsSync(passThroughOptionsCopy.destination || '') + ) { + continue; + } promises.push( limit(async () => { diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index af5b2d7c2..6280a5c44 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -30,6 +30,7 @@ import { TransferManager, Storage, DownloadResponse, + DownloadManyFilesOptions, } from '../src/index.js'; import assert from 'assert'; import * as path from 'path'; @@ -195,6 +196,10 @@ describe('Transfer Manager', () => { }); describe('downloadManyFiles', () => { + beforeEach(() => { + sandbox.stub(fs, 'existsSync').returns(true); + }); + it('calls download for each provided file', async () => { let count = 0; const firstFile = new File(bucket, 'first.txt'); @@ -276,6 +281,27 @@ describe('Transfer Manager', () => { await transferManager.downloadManyFiles([file], {passthroughOptions}); }); + it('does not download files that already exist locally when skipIfExists is true', async () => { + const firstFile = new File(bucket, 'first.txt'); + sandbox.stub(firstFile, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadManyFilesOptions).skipIfExists, + 0 + ); + }); + const secondFile = new File(bucket, 'second.txt'); + sandbox.stub(secondFile, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadManyFilesOptions).skipIfExists, + 0 + ); + }); + + const files = [firstFile, secondFile]; + const options = {skipIfExists: true}; + await transferManager.downloadManyFiles(files, options); + }); + it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => { const options = {}; const filename = 'first.txt';