Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: allow files in directories to be downloaded onto local machine #2199

Merged
merged 15 commits into from
May 21, 2024
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion src/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import * as resumableUpload from './resumable-upload';
import {Writable, Readable, pipeline, Transform, PassThrough} from 'stream';
import * as zlib from 'zlib';
import * as http from 'http';
import * as path from 'path';

import {
ExceptionMessages,
Expand Down Expand Up @@ -2097,7 +2098,16 @@ class File extends ServiceObject<File> {

const fileStream = this.createReadStream(options);
let receivedData = false;
if (destination) {

// Skip directory objects as they cannot be written to local filesystem
if (
destination &&
(destination.endsWith('/') || destination.endsWith('\\'))
) {
callback?.(null, Buffer.alloc(0));
vishwarajanand marked this conversation as resolved.
Show resolved Hide resolved
} else if (destination) {
fs.mkdirSync(path.dirname(destination), {recursive: true});
hochoy marked this conversation as resolved.
Show resolved Hide resolved
hochoy marked this conversation as resolved.
Show resolved Hide resolved
vishwarajanand marked this conversation as resolved.
Show resolved Hide resolved

fileStream
.on('error', callback)
.once('data', data => {
Expand Down
6 changes: 4 additions & 2 deletions src/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,9 @@ export class TransferManager {
*
* @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If
* a string is provided this will be treated as a GCS prefix and all files with that prefix will be downloaded.
* @param {DownloadManyFilesOptions} [options] Configuration options.
* @param {DownloadManyFilesOptions} [options] Configuration options. Setting options.prefix or options.stripPrefix
* or options.passthroughOptions.destination will cause the downloaded files to be written to the file system
* instead of being returned as a buffer.
* @returns {Promise<DownloadResponse[]>}
*
* @example
Expand Down Expand Up @@ -258,7 +260,7 @@ export class TransferManager {
{},
options.passthroughOptions
);
if (options.prefix) {
if (options.prefix || passThroughOptionsCopy.destination) {
vishwarajanand marked this conversation as resolved.
Show resolved Hide resolved
passThroughOptionsCopy.destination = path.join(
options.prefix || '',
passThroughOptionsCopy.destination || '',
Expand Down
71 changes: 71 additions & 0 deletions test/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import * as resumableUpload from '../src/resumable-upload';
import * as sinon from 'sinon';
import * as tmp from 'tmp';
import * as zlib from 'zlib';
import * as path from 'path';

import {
Bucket,
Expand Down Expand Up @@ -2555,6 +2556,12 @@ describe('File', () => {
});

describe('with destination', () => {
const sandbox = sinon.createSandbox();

afterEach(() => {
sandbox.restore();
});

it('should write the file to a destination if provided', done => {
tmp.setGracefulCleanup();
tmp.file((err, tmpFilePath) => {
Expand Down Expand Up @@ -2678,6 +2685,70 @@ describe('File', () => {
});
});
});

it('should recursively create directory and write file contents if destination path is nested', done => {
tmp.setGracefulCleanup();
tmp.dir(async (err, tmpDirPath) => {
assert.ifError(err);

const fileContents = 'nested-abcdefghijklmnopqrstuvwxyz';

Object.assign(fileReadStream, {
_read(this: Readable) {
this.push(fileContents);
this.push(null);
},
});

const nestedPath = path.join(tmpDirPath, 'a', 'b', 'c', 'file.txt');

file.download({destination: nestedPath}, (err: Error) => {
assert.ifError(err);
assert.strictEqual(fs.existsSync(nestedPath), true);
fs.readFile(nestedPath, (err, tmpFileContents) => {
assert.ifError(err);
assert.strictEqual(fileContents, tmpFileContents.toString());
done();
});
});
});
});

it('should skip write if asked to write a directory object', done => {
const mkdirSync = sandbox.spy(fakeFs, 'mkdirSync');
const createWriteStream = sandbox.spy(fakeFs, 'createWriteStream');

tmp.setGracefulCleanup();
tmp.dir(async (err, tmpDirPath) => {
assert.ifError(err);

const fileContents = '';

Object.assign(fileReadStream, {
_read(this: Readable) {
this.push(fileContents);
this.push(null);
},
});

const nestedDir = path.join(tmpDirPath, 'a', 'b', 'c', '/');

file.download(
{destination: nestedDir},
(err: Error, buffer: Buffer) => {
try {
assert.ifError(err);
assert.strictEqual(createWriteStream.callCount, 0);
assert.strictEqual(mkdirSync.callCount, 0);
assert.strictEqual(buffer.equals(Buffer.alloc(0)), true);
done();
} catch (e) {
done(e);
vishwarajanand marked this conversation as resolved.
Show resolved Hide resolved
}
}
);
});
});
});
});

Expand Down
29 changes: 29 additions & 0 deletions test/transfer-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,35 @@ describe('Transfer Manager', () => {
file.download = download;
await transferManager.downloadManyFiles([file], {stripPrefix});
});

it('sets the destination correctly when provided a passthroughOptions.destination', async () => {
const passthroughOptions = {
destination: 'test-destination',
};
const filename = 'first.txt';
const expectedDestination = path.normalize(
`${passthroughOptions.destination}/${filename}`
);
const download = (options: DownloadOptions) => {
assert.strictEqual(options.destination, expectedDestination);
};

const file = new File(bucket, filename);
file.download = download;
await transferManager.downloadManyFiles([file], {passthroughOptions});
});

it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => {
const options = {};
const filename = 'first.txt';
const download = (options: DownloadOptions) => {
assert.strictEqual(options.destination, undefined);
};

const file = new File(bucket, filename);
file.download = download;
await transferManager.downloadManyFiles([file], options);
});
});

describe('downloadFileInChunks', () => {
Expand Down