Skip to content
This repository has been archived by the owner on Jun 24, 2024. It is now read-only.

Commit

Permalink
build: upgrade gts and other dependencies for 2.x release (#220)
Browse files Browse the repository at this point in the history
  • Loading branch information
bcoe authored May 9, 2019
1 parent 1e60178 commit b39579a
Show file tree
Hide file tree
Showing 5 changed files with 336 additions and 276 deletions.
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,15 @@
"@types/pumpify": "^1.4.1",
"assert-rejects": "^1.0.0",
"codecov": "^3.0.4",
"gts": "^0.9.0",
"gts": "^1.0.0",
"intelli-espower-loader": "^1.0.1",
"is-stream": "^2.0.0",
"mocha": "^6.0.0",
"linkinator": "^1.1.2",
"mocha": "^6.1.4",
"mockery": "^2.1.0",
"nock": "^10.0.0",
"nyc": "^14.0.0",
"source-map-support": "^0.5.6",
"typescript": "~3.4.0",
"linkinator": "^1.1.2"
"typescript": "~3.4.0"
}
}
17 changes: 9 additions & 8 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,14 @@ import {upload} from '.';
const args = process.argv.slice(2);
const opts = {
bucket: args[0],
file: args[1]
file: args[1],
};

process.stdin.pipe(upload(opts))
.on('error', console.error)
.on('response', (resp, metadata) => {
if (!metadata || !metadata.mediaLink) return;
console.log('uploaded!');
console.log(metadata.mediaLink);
});
process.stdin
.pipe(upload(opts))
.on('error', console.error)
.on('response', (resp, metadata) => {
if (!metadata || !metadata.mediaLink) return;
console.log('uploaded!');
console.log(metadata.mediaLink);
});
82 changes: 53 additions & 29 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export interface ErrorWithCode extends Error {
code: number;
}

export type CreateUriCallback = (err: Error|null, uri?: string) => void;
export type CreateUriCallback = (err: Error | null, uri?: string) => void;

export interface Encryption {
key: {};
Expand Down Expand Up @@ -63,7 +63,7 @@ export interface UploadConfig {
* A customer-supplied encryption key. See
* https://cloud.google.com/storage/docs/encryption#customer-supplied.
*/
key?: string|Buffer;
key?: string | Buffer;

/**
* Resource name of the Cloud KMS key, of the form
Expand Down Expand Up @@ -93,8 +93,13 @@ export interface UploadConfig {
/**
* Apply a predefined set of access controls to the created file.
*/
predefinedAcl?: 'authenticatedRead'|'bucketOwnerFullControl'|
'bucketOwnerRead'|'private'|'projectPrivate'|'publicRead';
predefinedAcl?:
| 'authenticatedRead'
| 'bucketOwnerFullControl'
| 'bucketOwnerRead'
| 'private'
| 'projectPrivate'
| 'publicRead';

/**
* Make the uploaded file private. (Alias for config.predefinedAcl =
Expand Down Expand Up @@ -136,16 +141,21 @@ export interface ConfigMetadata {
export class Upload extends Pumpify {
bucket: string;
file: string;
authConfig?: {scopes?: string[];};
authConfig?: {scopes?: string[]};
authClient: GoogleAuth;
generation?: number;
key?: string|Buffer;
key?: string | Buffer;
kmsKeyName?: string;
metadata: ConfigMetadata;
offset?: number;
origin?: string;
predefinedAcl?: 'authenticatedRead'|'bucketOwnerFullControl'|
'bucketOwnerRead'|'private'|'projectPrivate'|'publicRead';
predefinedAcl?:
| 'authenticatedRead'
| 'bucketOwnerFullControl'
| 'bucketOwnerRead'
| 'private'
| 'projectPrivate'
| 'publicRead';
private?: boolean;
public?: boolean;
uri?: string;
Expand All @@ -155,7 +165,7 @@ export class Upload extends Pumpify {
uriProvidedManually: boolean;
numBytesWritten = 0;
numRetries = 0;
contentLength: number|'*';
contentLength: number | '*';
private bufferStream?: PassThrough;
private offsetStream?: PassThrough;

Expand All @@ -170,8 +180,9 @@ export class Upload extends Pumpify {
}

cfg.authConfig = cfg.authConfig || {};
cfg.authConfig.scopes =
['https://www.googleapis.com/auth/devstorage.full_control'];
cfg.authConfig.scopes = [
'https://www.googleapis.com/auth/devstorage.full_control',
];
this.authClient = cfg.authClient || new GoogleAuth(cfg.authConfig);

this.bucket = cfg.bucket;
Expand All @@ -192,7 +203,9 @@ export class Upload extends Pumpify {
const base64Key = Buffer.from(cfg.key as string).toString('base64');
this.encryption = {
key: base64Key,
hash: createHash('sha256').update(cfg.key).digest('base64')
hash: createHash('sha256')
.update(cfg.key)
.digest('base64'),
};
}

Expand All @@ -208,8 +221,9 @@ export class Upload extends Pumpify {
this.numBytesWritten = 0;
this.numRetries = 0;

const contentLength =
cfg.metadata ? Number(cfg.metadata.contentLength) : NaN;
const contentLength = cfg.metadata
? Number(cfg.metadata.contentLength)
: NaN;
this.contentLength = isNaN(contentLength) ? '*' : contentLength;

this.once('writing', () => {
Expand All @@ -228,7 +242,7 @@ export class Upload extends Pumpify {

createURI(): Promise<string>;
createURI(callback: CreateUriCallback): void;
createURI(callback?: CreateUriCallback): void|Promise<string> {
createURI(callback?: CreateUriCallback): void | Promise<string> {
if (!callback) {
return this.createURIAsync();
}
Expand All @@ -243,12 +257,13 @@ export class Upload extends Pumpify {
url: [BASE_URI, this.bucket, 'o'].join('/'),
params: {name: this.file, uploadType: 'resumable'},
data: metadata,
headers: {}
headers: {},
};

if (metadata.contentLength) {
reqOpts.headers!['X-Upload-Content-Length'] =
metadata.contentLength.toString();
reqOpts.headers![
'X-Upload-Content-Length'
] = metadata.contentLength.toString();
}

if (metadata.contentType) {
Expand Down Expand Up @@ -297,8 +312,9 @@ export class Upload extends Pumpify {
// The offset stream allows us to analyze each incoming
// chunk to analyze it against what the upstream API already
// has stored for this upload.
const offsetStream = this.offsetStream =
new Transform({transform: this.onChunk.bind(this)});
const offsetStream = (this.offsetStream = new Transform({
transform: this.onChunk.bind(this),
}));

// The delay stream gives us a chance to catch the response
// from the API request before we signal to the user that
Expand Down Expand Up @@ -350,7 +366,7 @@ export class Upload extends Pumpify {
method: 'PUT',
url: this.uri,
headers: {
'Content-Range': 'bytes ' + this.offset + '-*/' + this.contentLength
'Content-Range': 'bytes ' + this.offset + '-*/' + this.contentLength,
},
body: requestStreamEmbeddedStream,
};
Expand All @@ -363,13 +379,16 @@ export class Upload extends Pumpify {
}

private onChunk(
chunk: string, enc: string, next: (err?: Error, data?: string) => void) {
chunk: string,
enc: string,
next: (err?: Error, data?: string) => void
) {
const offset = this.offset!;
const numBytesWritten = this.numBytesWritten;

this.emit('progress', {
bytesWritten: this.numBytesWritten,
contentLength: this.contentLength
contentLength: this.contentLength,
});

// check if this is the same content uploaded previously. this caches a
Expand Down Expand Up @@ -411,7 +430,7 @@ export class Upload extends Pumpify {
const opts: GaxiosOptions = {
method: 'PUT',
url: this.uri!,
headers: {'Content-Length': 0, 'Content-Range': 'bytes */*'}
headers: {'Content-Length': 0, 'Content-Range': 'bytes */*'},
};
try {
const resp = await this.makeRequest(opts);
Expand Down Expand Up @@ -453,8 +472,9 @@ export class Upload extends Pumpify {
reqOpts.headers = reqOpts.headers || {};
reqOpts.headers['x-goog-encryption-algorithm'] = 'AES256';
reqOpts.headers['x-goog-encryption-key'] = this.encryption.key.toString();
reqOpts.headers['x-goog-encryption-key-sha256'] =
this.encryption.hash.toString();
reqOpts.headers[
'x-goog-encryption-key-sha256'
] = this.encryption.hash.toString();
}

if (this.userProject) {
Expand All @@ -469,8 +489,10 @@ export class Upload extends Pumpify {
}
// If no error was returned, but the response had an invalid status
// code, create a new error to be passed to the callback.
if ((res.status < 200 || res.status >= 300) &&
res.status !== RESUMABLE_INCOMPLETE_STATUS_CODE) {
if (
(res.status < 200 || res.status >= 300) &&
res.status !== RESUMABLE_INCOMPLETE_STATUS_CODE
) {
const e = new Error(`The request failed with a ${res.status}.`);
(e as ErrorWithCode).code = res.status;
}
Expand Down Expand Up @@ -555,7 +577,9 @@ export function upload(cfg: UploadConfig) {
export function createURI(cfg: UploadConfig): Promise<string>;
export function createURI(cfg: UploadConfig, callback: CreateUriCallback): void;
export function createURI(
cfg: UploadConfig, callback?: CreateUriCallback): void|Promise<string> {
cfg: UploadConfig,
callback?: CreateUriCallback
): void | Promise<string> {
const up = new Upload(cfg);
if (!callback) {
return up.createURI();
Expand Down
100 changes: 51 additions & 49 deletions system-test/kitchen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,83 +13,85 @@ import {createURI, upload} from '../src';
const bucketName = process.env.BUCKET_NAME || 'gcs-resumable-upload-test';

describe('end to end', () => {
it('should work', (done) => {
it('should work', done => {
let uploadSucceeded = false;
fs.createReadStream('daw.jpg')
.on('error', done)
.pipe(upload({
.on('error', done)
.pipe(
upload({
bucket: bucketName,
file: 'daw.jpg',
metadata: {contentType: 'image/jpg'}
}))
.on('error', done)
.on('response',
(resp) => {
uploadSucceeded = resp.status === 200;
})
.on('finish', () => {
assert.strictEqual(uploadSucceeded, true);
done();
});
metadata: {contentType: 'image/jpg'},
})
)
.on('error', done)
.on('response', resp => {
uploadSucceeded = resp.status === 200;
})
.on('finish', () => {
assert.strictEqual(uploadSucceeded, true);
done();
});
});

it('should resume an interrupted upload', (done) => {
it('should resume an interrupted upload', done => {
fs.stat('daw.jpg', (err, fd) => {
assert.ifError(err);

const size = fd.size;

// tslint:disable-next-line no-any
type DoUploadCallback = (...args: any[]) => void;
const doUpload =
(opts: {interrupt?: boolean}, callback: DoUploadCallback) => {
let sizeStreamed = 0;
let destroyed = false;
const doUpload = (
opts: {interrupt?: boolean},
callback: DoUploadCallback
) => {
let sizeStreamed = 0;
let destroyed = false;

const ws = upload({
bucket: bucketName,
file: 'daw.jpg',
metadata: {contentType: 'image/jpg'}
});
const ws = upload({
bucket: bucketName,
file: 'daw.jpg',
metadata: {contentType: 'image/jpg'},
});

fs.createReadStream('daw.jpg')
.on('error', callback)
.on('data',
function(this: Readable, chunk) {
sizeStreamed += chunk.length;
fs.createReadStream('daw.jpg')
.on('error', callback)
.on('data', function(this: Readable, chunk) {
sizeStreamed += chunk.length;

if (!destroyed && opts.interrupt &&
sizeStreamed >= size / 2) {
// stop sending data half way through
destroyed = true;
this.destroy();
ws.destroy(new Error('Interrupted'));
}
})
.pipe(ws)
.on('error', callback)
.on('metadata', callback.bind(null, null));
};
if (!destroyed && opts.interrupt && sizeStreamed >= size / 2) {
// stop sending data half way through
destroyed = true;
this.destroy();
ws.destroy(new Error('Interrupted'));
}
})
.pipe(ws)
.on('error', callback)
.on('metadata', callback.bind(null, null));
};

doUpload({interrupt: true}, (err: Error) => {
assert.strictEqual(err.message, 'Interrupted');

doUpload({interrupt: false}, (err: Error, metadata: {size: number}) => {
assert.ifError(err);
assert.equal(metadata.size, size);
assert.strictEqual(Number(metadata.size), size);
done();
});
});
});
});

it('should just make an upload URI', (done) => {
it('should just make an upload URI', done => {
createURI(
{
bucket: bucketName,
file: 'daw.jpg',
metadata: {contentType: 'image/jpg'}
},
done);
{
bucket: bucketName,
file: 'daw.jpg',
metadata: {contentType: 'image/jpg'},
},
done
);
});
});
Loading

0 comments on commit b39579a

Please sign in to comment.