From 14f287c8c6e0fae1ed56d77bbbd76394902a5091 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 19 May 2022 09:36:05 -0700 Subject: [PATCH] feat!: Use `HashStreamValidator` for Data Validation (#1951) * feat!: Use `HashStreamValidator` for stream validation - Removes `fast-crc32c` support * fix: remove logs * test: `crc32cGenerator` option * test: improve `compression` test reliability * chore: add temp debug logs * chore: more debug logs * fix: use static gzip content to remove test variance * chore: remove debug logs * chore: lint --- package.json | 1 - src/bucket.ts | 10 +- src/crc32c.ts | 4 + src/file.ts | 66 +++++----- src/hash-stream-validator.ts | 96 ++++++++++++++ src/index.ts | 1 + src/storage.ts | 14 ++- test/bucket.ts | 14 ++- test/file.ts | 235 ++++++++++++++++------------------- test/index.ts | 16 ++- 10 files changed, 284 insertions(+), 173 deletions(-) create mode 100644 src/hash-stream-validator.ts diff --git a/package.json b/package.json index 42e466709..52ef62571 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,6 @@ "extend": "^3.0.2", "gaxios": "^5.0.0", "google-auth-library": "^8.0.1", - "hash-stream-validation": "^0.2.2", "mime": "^3.0.0", "mime-types": "^2.0.8", "p-limit": "^3.0.1", diff --git a/src/bucket.ts b/src/bucket.ts index 3c5d1b8b9..2ba1c41d1 100644 --- a/src/bucket.ts +++ b/src/bucket.ts @@ -63,6 +63,7 @@ import { Query, } from './signer'; import {Readable} from 'stream'; +import {CRC32CValidatorGenerator} from './crc32c'; interface SourceObject { name: string; @@ -623,6 +624,7 @@ class Bucket extends ServiceObject { acl: Acl; iam: Iam; + crc32cGenerator: CRC32CValidatorGenerator; // eslint-disable-next-line @typescript-eslint/no-unused-vars getFilesStream(query?: GetFilesOptions): Readable { @@ -1035,6 +1037,9 @@ class Bucket extends ServiceObject { pathPrefix: '/defaultObjectAcl', }); + this.crc32cGenerator = + options.crc32cGenerator || this.storage.crc32cGenerator; + this.iam = new Iam(this); this.getFilesStream = paginator.streamify('getFiles'); @@ -3718,11 +3723,6 @@ class Bucket extends ServiceObject { * * Resumable uploads are enabled by default * - * For faster crc32c computation, you must manually install - * {@link https://www.npmjs.com/package/fast-crc32c| `fast-crc32c`}: - * - * $ npm install --save fast-crc32c - * * See {@link https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload#uploads| Upload Options (Simple or Resumable)} * See {@link https://cloud.google.com/storage/docs/json_api/v1/objects/insert| Objects: insert API Documentation} * diff --git a/src/crc32c.ts b/src/crc32c.ts index 8e4fcad8a..f8d6c10aa 100644 --- a/src/crc32c.ts +++ b/src/crc32c.ts @@ -117,6 +117,9 @@ interface CRC32CValidatorGenerator { (): CRC32CValidator; } +const CRC32C_DEFAULT_VALIDATOR_GENERATOR: CRC32CValidatorGenerator = () => + new CRC32C(); + const CRC32C_EXCEPTION_MESSAGES = { INVALID_INIT_BASE64_RANGE: (l: number) => `base64-encoded data expected to equal 4 bytes, not ${l}`, @@ -305,6 +308,7 @@ class CRC32C implements CRC32CValidator { export { CRC32C, + CRC32C_DEFAULT_VALIDATOR_GENERATOR, CRC32C_EXCEPTION_MESSAGES, CRC32C_EXTENSIONS, CRC32C_EXTENSION_TABLE, diff --git a/src/file.ts b/src/file.ts index 4f247bbe8..329b43b37 100644 --- a/src/file.ts +++ b/src/file.ts @@ -27,8 +27,6 @@ import compressible = require('compressible'); import * as crypto from 'crypto'; import * as extend from 'extend'; import * as fs from 'fs'; -// eslint-disable-next-line @typescript-eslint/no-var-requires -const hashStreamValidation = require('hash-stream-validation'); import * as mime from 'mime'; // eslint-disable-next-line @typescript-eslint/no-var-requires const pumpify = require('pumpify'); @@ -68,6 +66,9 @@ import { unicodeJSONStringify, formatAsUTCISO, } from './util'; +import {CRC32CValidatorGenerator} from './crc32c'; +import {HashStreamValidator} from './hash-stream-validator'; + import retry = require('async-retry'); export type GetExpirationDateResponse = [Date]; @@ -288,11 +289,12 @@ export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com'; const GS_URL_REGEXP = /^gs:\/\/([a-z0-9_.-]+)\/(.+)$/; export interface FileOptions { + crc32cGenerator?: CRC32CValidatorGenerator; encryptionKey?: string | Buffer; generation?: number | string; kmsKeyName?: string; - userProject?: string; preconditionOpts?: PreconditionOptions; + userProject?: string; } export interface CopyOptions { @@ -419,7 +421,7 @@ export enum FileExceptionMessages { */ class File extends ServiceObject { acl: Acl; - + crc32cGenerator: CRC32CValidatorGenerator; bucket: Bucket; storage: Storage; kmsKeyName?: string; @@ -874,6 +876,9 @@ class File extends ServiceObject { pathPrefix: '/acl', }); + this.crc32cGenerator = + options.crc32cGenerator || this.bucket.crc32cGenerator; + this.instanceRetryValue = this.storage?.retryOptions?.autoRetry; this.instancePreconditionOpts = options?.preconditionOpts; } @@ -1209,11 +1214,6 @@ class File extends ServiceObject { * code "CONTENT_DOWNLOAD_MISMATCH". If you receive this error, the best * recourse is to try downloading the file again. * - * For faster crc32c computation, you must manually install - * {@link https://www.npmjs.com/package/fast-crc32c| `fast-crc32c`}: - * - * $ npm install --save fast-crc32c - * * NOTE: Readable streams will emit the `end` event when the file is fully * downloaded. * @@ -1277,8 +1277,7 @@ class File extends ServiceObject { typeof options.start === 'number' || typeof options.end === 'number'; const tailRequest = options.end! < 0; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let validateStream: any; // Created later, if necessary. + let validateStream: HashStreamValidator | undefined = undefined; const throughStream = streamEvents(new PassThrough()); @@ -1287,12 +1286,10 @@ class File extends ServiceObject { let md5 = false; if (typeof options.validation === 'string') { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (options as any).validation = ( - options.validation as string - ).toLowerCase(); - crc32c = options.validation === 'crc32c'; - md5 = options.validation === 'md5'; + const value = options.validation.toLowerCase().trim(); + + crc32c = value === 'crc32c'; + md5 = value === 'md5'; } else if (options.validation === false) { crc32c = false; } @@ -1406,7 +1403,12 @@ class File extends ServiceObject { }); } - validateStream = hashStreamValidation({crc32c, md5}); + validateStream = new HashStreamValidator({ + crc32c, + md5, + crc32cGenerator: this.crc32cGenerator, + }); + throughStreams.push(validateStream); } @@ -1474,15 +1476,14 @@ class File extends ServiceObject { // the best. let failed = crc32c || md5; - if (crc32c && hashes.crc32c) { - // We must remove the first four bytes from the returned checksum. - // http://stackoverflow.com/questions/25096737/ - // base64-encoding-of-crc32c-long-value - failed = !validateStream.test('crc32c', hashes.crc32c.substr(4)); - } + if (validateStream) { + if (crc32c && hashes.crc32c) { + failed = !validateStream.test('crc32c', hashes.crc32c); + } - if (md5 && hashes.md5) { - failed = !validateStream.test('md5', hashes.md5); + if (md5 && hashes.md5) { + failed = !validateStream.test('md5', hashes.md5); + } } if (md5 && !hashes.md5) { @@ -1730,11 +1731,6 @@ class File extends ServiceObject { * resumable feature is disabled. *

* - * For faster crc32c computation, you must manually install - * {@link https://www.npmjs.com/package/fast-crc32c| `fast-crc32c`}: - * - * $ npm install --save fast-crc32c - * * NOTE: Writable streams will emit the `finish` event when the file is fully * uploaded. * @@ -1846,9 +1842,10 @@ class File extends ServiceObject { // Collect data as it comes in to store in a hash. This is compared to the // checksum value on the returned metadata from the API. - const validateStream = hashStreamValidation({ + const validateStream = new HashStreamValidator({ crc32c, md5, + crc32cGenerator: this.crc32cGenerator, }); const fileWriteStream = duplexify(); @@ -1896,10 +1893,7 @@ class File extends ServiceObject { let failed = crc32c || md5; if (crc32c && metadata.crc32c) { - // We must remove the first four bytes from the returned checksum. - // http://stackoverflow.com/questions/25096737/ - // base64-encoding-of-crc32c-long-value - failed = !validateStream.test('crc32c', metadata.crc32c.substr(4)); + failed = !validateStream.test('crc32c', metadata.crc32c); } if (md5 && metadata.md5Hash) { diff --git a/src/hash-stream-validator.ts b/src/hash-stream-validator.ts new file mode 100644 index 000000000..db4a527f4 --- /dev/null +++ b/src/hash-stream-validator.ts @@ -0,0 +1,96 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {createHash, Hash} from 'crypto'; +import {Transform} from 'stream'; + +import { + CRC32CValidatorGenerator, + CRC32C_DEFAULT_VALIDATOR_GENERATOR, + CRC32CValidator, +} from './crc32c'; + +interface HashStreamValidatorOptions { + crc32c: boolean; + md5: boolean; + crc32cGenerator: CRC32CValidatorGenerator; +} + +class HashStreamValidator extends Transform { + readonly crc32cEnabled: boolean; + readonly md5Enabled: boolean; + + #crc32cHash?: CRC32CValidator = undefined; + #md5Hash?: Hash = undefined; + + #md5Digest = ''; + + constructor(options: Partial = {}) { + super(); + + this.crc32cEnabled = !!options.crc32c; + this.md5Enabled = !!options.md5; + + if (this.crc32cEnabled) { + const crc32cGenerator = + options.crc32cGenerator || CRC32C_DEFAULT_VALIDATOR_GENERATOR; + + this.#crc32cHash = crc32cGenerator(); + } + + if (this.md5Enabled) { + this.#md5Hash = createHash('md5'); + } + } + + _flush(callback: () => void) { + if (this.#md5Hash) { + this.#md5Digest = this.#md5Hash.digest('base64'); + } + + callback(); + } + + _transform( + chunk: Buffer, + encoding: BufferEncoding, + callback: (e?: Error) => void + ) { + this.push(chunk, encoding); + + try { + if (this.#crc32cHash) this.#crc32cHash.update(chunk); + if (this.#md5Hash) this.#md5Hash.update(chunk); + callback(); + } catch (e) { + callback(e as Error); + } + } + + test(hash: 'crc32c' | 'md5', sum: Buffer | string): boolean { + const check = Buffer.isBuffer(sum) ? sum.toString('base64') : sum; + + if (hash === 'crc32c' && this.#crc32cHash) { + return this.#crc32cHash.validate(check); + } + + if (hash === 'md5' && this.#md5Hash) { + return this.#md5Digest === check; + } + + return false; + } +} + +export {HashStreamValidator, HashStreamValidatorOptions}; diff --git a/src/index.ts b/src/index.ts index e0e918143..0f8210aa9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -197,6 +197,7 @@ export { SetStorageClassResponse, SignedPostPolicyV4Output, } from './file'; +export * from './hash-stream-validator'; export { HmacKey, HmacKeyMetadata, diff --git a/src/storage.ts b/src/storage.ts index bf068e034..233fdeee6 100644 --- a/src/storage.ts +++ b/src/storage.ts @@ -23,6 +23,10 @@ import {Channel} from './channel'; import {File} from './file'; import {normalize} from './util'; import {HmacKey, HmacKeyMetadata, HmacKeyOptions} from './hmacKey'; +import { + CRC32CValidatorGenerator, + CRC32C_DEFAULT_VALIDATOR_GENERATOR, +} from './crc32c'; export interface GetServiceAccountOptions { userProject?: string; @@ -68,18 +72,20 @@ export interface PreconditionOptions { } export interface StorageOptions extends ServiceOptions { - retryOptions?: RetryOptions; /** * The API endpoint of the service used to make requests. * Defaults to `storage.googleapis.com`. */ apiEndpoint?: string; + crc32cGenerator?: CRC32CValidatorGenerator; + retryOptions?: RetryOptions; } export interface BucketOptions { + crc32cGenerator?: CRC32CValidatorGenerator; kmsKeyName?: string; - userProject?: string; preconditionOpts?: PreconditionOptions; + userProject?: string; } export interface Cors { @@ -457,6 +463,8 @@ export class Storage extends Service { */ acl: typeof Storage.acl; + crc32cGenerator: CRC32CValidatorGenerator; + getBucketsStream(): Readable { // placeholder body, overwritten in constructor return new Readable(); @@ -624,6 +632,8 @@ export class Storage extends Service { * @see Storage.acl */ this.acl = Storage.acl; + this.crc32cGenerator = + options.crc32cGenerator || CRC32C_DEFAULT_VALIDATOR_GENERATOR; this.retryOptions = config.retryOptions; diff --git a/test/bucket.ts b/test/bucket.ts index 29e28b51f..fedd5bfe8 100644 --- a/test/bucket.ts +++ b/test/bucket.ts @@ -30,7 +30,7 @@ import * as path from 'path'; import * as proxyquire from 'proxyquire'; import * as stream from 'stream'; -import {Bucket, Channel, Notification} from '../src'; +import {Bucket, Channel, Notification, CRC32C} from '../src'; import { CreateWriteStreamOptions, File, @@ -197,6 +197,7 @@ describe('Bucket', () => { }, idempotencyStrategy: IdempotencyStrategy.RetryConditional, }, + crc32cGenerator: () => new CRC32C(), }; const BUCKET_NAME = 'test-bucket'; @@ -429,6 +430,17 @@ describe('Bucket', () => { assert.strictEqual(bucket.userProject, fakeUserProject); }); + + it('should accept a `crc32cGenerator`', () => { + const crc32cGenerator = () => {}; + + const bucket = new Bucket(STORAGE, 'bucket-name', {crc32cGenerator}); + assert.strictEqual(bucket.crc32cGenerator, crc32cGenerator); + }); + + it("should use storage's `crc32cGenerator` by default", () => { + assert.strictEqual(bucket.crc32cGenerator, STORAGE.crc32cGenerator); + }); }); describe('addLifecycleRule', () => { diff --git a/test/file.ts b/test/file.ts index 40bee1ea9..171b88020 100644 --- a/test/file.ts +++ b/test/file.ts @@ -44,6 +44,7 @@ import { SetFileMetadataOptions, GetSignedUrlConfig, GenerateSignedPostPolicyV2Options, + CRC32C, } from '../src'; import { SignedPostPolicyV4Output, @@ -112,13 +113,6 @@ const fakeZlib = extend(true, {}, zlib, { }, }); -let hashStreamValidationOverride: Function | null; -// eslint-disable-next-line @typescript-eslint/no-var-requires -const hashStreamValidation = require('hash-stream-validation'); -function fakeHashStreamValidation(...args: Array<{}>) { - return (hashStreamValidationOverride || hashStreamValidation)(...args); -} - // eslint-disable-next-line @typescript-eslint/no-var-requires const osCached = extend(true, {}, require('os')); const fakeOs = extend(true, {}, osCached); @@ -184,6 +178,19 @@ describe('File', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any let BUCKET: any; + const DATA = 'test data'; + // crc32c hash of 'test data' + const CRC32C_HASH = 'M3m0yg=='; + // md5 hash of 'test data' + const MD5_HASH = '63M6AMDJ0zbmVpGjerVCkw=='; + // crc32c hash of `zlib.gzipSync(Buffer.from(DATA), {level: 9})` + const GZIPPED_DATA = Buffer.from( + 'H4sIAAAAAAACEytJLS5RSEksSQQAsq4I0wkAAAA=', + 'base64' + ); + //crc32c hash of `GZIPPED_DATA` + const CRC32C_HASH_GZIP = '64jygg=='; + before(() => { File = proxyquire('../src/file.js', { './nodejs-common': { @@ -193,7 +200,6 @@ describe('File', () => { '@google-cloud/promisify': fakePromisify, fs: fakeFs, '../src/resumable-upload': fakeResumableUpload, - 'hash-stream-validation': fakeHashStreamValidation, os: fakeOs, './signer': fakeSigner, zlib: fakeZlib, @@ -245,7 +251,6 @@ describe('File', () => { createGunzipOverride = null; handleRespOverride = null; - hashStreamValidationOverride = null; makeWritableStreamOverride = null; resumableUploadOverride = null; }); @@ -433,6 +438,17 @@ describe('File', () => { new File(BUCKET, FILE_NAME, {encryptionKey: key}); }); + it('should accept a `crc32cGenerator`', () => { + const crc32cGenerator = () => {}; + + const file = new File(BUCKET, 'name', {crc32cGenerator}); + assert.strictEqual(file.crc32cGenerator, crc32cGenerator); + }); + + it("should use the bucket's `crc32cGenerator` by default", () => { + assert.strictEqual(file.crc32cGenerator, BUCKET.crc32cGenerator); + }); + describe('userProject', () => { const USER_PROJECT = 'grapce-spaceship-123'; @@ -854,9 +870,6 @@ describe('File', () => { }); describe('createReadStream', () => { - const CRC32C_HASH = 'crc32c-hash'; - const MD5_HASH = 'md5-hash'; - function getFakeRequest(data?: {}) { let requestOptions: DecorateRequestOptions | undefined; @@ -1235,14 +1248,7 @@ describe('File', () => { }); describe('compression', () => { - const DATA = 'test data'; - const GZIPPED_DATA = zlib.gzipSync(DATA); - beforeEach(() => { - hashStreamValidationOverride = () => - Object.assign(new PassThrough(), { - test: () => true, - }); handleRespOverride = ( err: Error, res: {}, @@ -1255,39 +1261,39 @@ describe('File', () => { return { headers: { 'content-encoding': 'gzip', - 'x-goog-hash': `crc32c=${CRC32C_HASH},md5=${MD5_HASH}`, + 'x-goog-hash': `crc32c=${CRC32C_HASH_GZIP},md5=${MD5_HASH}`, }, }; }, }); callback(null, null, rawResponseStream); - setImmediate(() => { - rawResponseStream.end(GZIPPED_DATA); - }); + + rawResponseStream.end(GZIPPED_DATA); }; file.requestStream = getFakeSuccessfulRequest(GZIPPED_DATA); }); - it('should gunzip the response', done => { - file - .createReadStream() - .once('error', done) - .on('data', (data: {}) => { - assert.strictEqual(data.toString(), DATA); - done(); - }) - .resume(); + it('should gunzip the response', async () => { + const collection: Buffer[] = []; + + for await (const data of file.createReadStream()) { + collection.push(data); + } + + assert.equal(Buffer.concat(collection).toString(), DATA); }); - it('should not gunzip the response if "decompress: false" is passed', done => { - file - .createReadStream({decompress: false}) - .once('error', done) - .on('data', (data: {}) => { - assert.strictEqual(data, GZIPPED_DATA); - done(); - }) - .resume(); + it('should not gunzip the response if "decompress: false" is passed', async () => { + const collection: Buffer[] = []; + + for await (const data of file.createReadStream({decompress: false})) { + collection.push(data); + } + + assert.equal( + Buffer.compare(Buffer.concat(collection), GZIPPED_DATA), + 0 + ); }); it('should emit errors from the gunzip stream', done => { @@ -1332,17 +1338,15 @@ describe('File', () => { }); describe('validation', () => { - const data = 'test'; - let fakeValidationStream: Stream & {test: Function}; + let responseCRC32C = CRC32C_HASH; + let responseMD5 = MD5_HASH; beforeEach(() => { - file.getMetadata = () => Promise.resolve({}); - fakeValidationStream = Object.assign(new PassThrough(), { - test: () => true, - }); - hashStreamValidationOverride = () => { - return fakeValidationStream; - }; + responseCRC32C = CRC32C_HASH; + responseMD5 = MD5_HASH; + + file.getMetadata = async () => ({}); + handleRespOverride = ( err: Error, res: {}, @@ -1354,75 +1358,72 @@ describe('File', () => { toJSON() { return { headers: { - 'x-goog-hash': `crc32c=${CRC32C_HASH},md5=${MD5_HASH}`, + 'x-goog-hash': `crc32c=${responseCRC32C},md5=${responseMD5}`, }, }; }, }); callback(null, null, rawResponseStream); setImmediate(() => { - rawResponseStream.end(data); + rawResponseStream.end(DATA); }); }; - file.requestStream = getFakeSuccessfulRequest(data); + file.requestStream = getFakeSuccessfulRequest(DATA); }); + function setFileValidationToError(e: Error = new Error('test-error')) { + // Simulating broken CRC32C instance - used by the validation stream + file.crc32cGenerator = () => { + class C extends CRC32C { + update() { + throw e; + } + } + + return new C(); + }; + } + describe('server decompression', () => { it('should skip validation if file was stored compressed', done => { + file.metadata.crc32c = '.invalid.'; file.metadata.contentEncoding = 'gzip'; - const validateStub = sinon.stub().returns(true); - fakeValidationStream.test = validateStub; - file .createReadStream({validation: 'crc32c'}) .on('error', done) - .on('end', () => { - assert(validateStub.notCalled); - done(); - }) + .on('end', done) .resume(); }); }); it('should emit errors from the validation stream', done => { - const error = new Error('Error.'); + const expectedError = new Error('test error'); - hashStreamValidationOverride = () => { - setImmediate(() => { - fakeValidationStream.emit('error', error); - }); - return fakeValidationStream; - }; - - file.requestStream = getFakeSuccessfulRequest(data); + file.requestStream = getFakeSuccessfulRequest(DATA); + setFileValidationToError(expectedError); file .createReadStream() .on('error', (err: Error) => { - assert.strictEqual(err, error); + assert(err === expectedError); + done(); }) .resume(); }); it('should not handle both error and end events', done => { - const error = new Error('Error.'); + const expectedError = new Error('test error'); - hashStreamValidationOverride = () => { - setImmediate(() => { - fakeValidationStream.emit('error', error); - }); - return fakeValidationStream; - }; - - file.requestStream = getFakeSuccessfulRequest(data); + file.requestStream = getFakeSuccessfulRequest(DATA); + setFileValidationToError(expectedError); file .createReadStream() .on('error', (err: Error) => { - assert.strictEqual(err, error); - fakeValidationStream.emit('end'); + assert(err === expectedError); + setImmediate(done); }) .on('end', () => { @@ -1439,10 +1440,12 @@ describe('File', () => { file.getMetadata = (options: GetFileMetadataOptions) => { assert.strictEqual(options.userProject, fakeOptions.userProject); setImmediate(done); - return Promise.resolve({}); + return Promise.resolve({ + crc32c: CRC32C_HASH, + }); }; - file.requestStream = getFakeSuccessfulRequest('data'); + file.requestStream = getFakeSuccessfulRequest(DATA); file.createReadStream(fakeOptions).on('error', done).resume(); }); @@ -1453,7 +1456,7 @@ describe('File', () => { return Promise.reject(error); }; - file.requestStream = getFakeSuccessfulRequest('data'); + file.requestStream = getFakeSuccessfulRequest(DATA); file .createReadStream() @@ -1465,13 +1468,8 @@ describe('File', () => { }); it('should validate with crc32c', done => { - file.requestStream = getFakeSuccessfulRequest(data); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = (algo: string, value: string) => { - assert.strictEqual(algo, 'crc32c'); - assert.strictEqual(value, CRC32C_HASH.substr(4)); - return true; - }; + file.requestStream = getFakeSuccessfulRequest(DATA); + file .createReadStream({validation: 'crc32c'}) .on('error', done) @@ -1481,8 +1479,9 @@ describe('File', () => { it('should emit an error if crc32c validation fails', done => { file.requestStream = getFakeSuccessfulRequest('bad-data'); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = () => false; + + responseCRC32C = 'bad-crc32c'; + file .createReadStream({validation: 'crc32c'}) .on('error', (err: ApiError) => { @@ -1493,13 +1492,8 @@ describe('File', () => { }); it('should validate with md5', done => { - file.requestStream = getFakeSuccessfulRequest(data); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = (algo: string, value: string) => { - assert.strictEqual(algo, 'md5'); - assert.strictEqual(value, MD5_HASH); - return true; - }; + file.requestStream = getFakeSuccessfulRequest(DATA); + file .createReadStream({validation: 'md5'}) .on('error', done) @@ -1509,11 +1503,9 @@ describe('File', () => { it('should emit an error if md5 validation fails', done => { file.requestStream = getFakeSuccessfulRequest('bad-data'); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = (algo: string) => { - assert.strictEqual(algo, 'md5'); - return false; - }; + + responseMD5 = 'bad-md5'; + file .createReadStream({validation: 'md5'}) .on('error', (err: ApiError) => { @@ -1525,11 +1517,9 @@ describe('File', () => { it('should default to crc32c validation', done => { file.requestStream = getFakeSuccessfulRequest('bad-data'); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = (algo: string) => { - assert.strictEqual(algo, 'crc32c'); - return false; - }; + + responseCRC32C = 'bad-crc32c'; + file .createReadStream() .on('error', (err: ApiError) => { @@ -1540,9 +1530,9 @@ describe('File', () => { }); it('should ignore a data mismatch if validation: false', done => { - file.requestStream = getFakeSuccessfulRequest(data); + file.requestStream = getFakeSuccessfulRequest(DATA); // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = () => false; + // (fakeValidationStream as any).test = () => false; file .createReadStream({validation: false}) .resume() @@ -1569,30 +1559,21 @@ describe('File', () => { }); callback(null, null, rawResponseStream); setImmediate(() => { - rawResponseStream.end(data); + rawResponseStream.end(DATA); }); }; - file.requestStream = getFakeSuccessfulRequest(data); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = (algo: string, value: string) => { - assert.strictEqual(algo, 'crc32c'); - assert.strictEqual(value, CRC32C_HASH.substr(4)); - return true; - }; + file.requestStream = getFakeSuccessfulRequest(DATA); file.createReadStream().on('error', done).on('end', done).resume(); }); describe('destroying the through stream', () => { - beforeEach(() => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (fakeValidationStream as any).test = () => false; - }); - it('should destroy after failed validation', done => { file.requestStream = getFakeSuccessfulRequest('bad-data'); + responseMD5 = 'bad-md5'; + const readStream = file.createReadStream({validation: 'md5'}); readStream.destroy = (err: ApiError) => { assert.strictEqual(err.code, 'CONTENT_DOWNLOAD_MISMATCH'); @@ -1698,7 +1679,7 @@ describe('File', () => { }); it('should end the through stream', done => { - file.requestStream = getFakeSuccessfulRequest('body'); + file.requestStream = getFakeSuccessfulRequest(DATA); const readStream = file.createReadStream({start: 100}); readStream.end = done; @@ -2210,7 +2191,7 @@ describe('File', () => { const data = 'test'; const fakeMetadata = { - crc32c: {crc32c: '####wA=='}, + crc32c: {crc32c: 'hqBywA=='}, md5: {md5Hash: 'CY9rzUYh03PK3k6DJie09g=='}, }; diff --git a/test/index.ts b/test/index.ts index 80d59b877..8283b41e6 100644 --- a/test/index.ts +++ b/test/index.ts @@ -26,7 +26,7 @@ import * as assert from 'assert'; import {describe, it, before, beforeEach, after, afterEach} from 'mocha'; import * as proxyquire from 'proxyquire'; // eslint-disable-next-line @typescript-eslint/no-unused-vars -import {Bucket} from '../src'; +import {Bucket, CRC32C_DEFAULT_VALIDATOR_GENERATOR} from '../src'; import {GetFilesOptions} from '../src/bucket'; import sinon = require('sinon'); import {HmacKey} from '../src/hmacKey'; @@ -398,6 +398,20 @@ describe('Storage', () => { assert.strictEqual(calledWith.apiEndpoint, 'https://some.fake.endpoint'); }); + it('should accept a `crc32cGenerator`', () => { + const crc32cGenerator = () => {}; + + const storage = new Storage({crc32cGenerator}); + assert.strictEqual(storage.crc32cGenerator, crc32cGenerator); + }); + + it('should use `CRC32C_DEFAULT_VALIDATOR_GENERATOR` by default', () => { + assert.strictEqual( + storage.crc32cGenerator, + CRC32C_DEFAULT_VALIDATOR_GENERATOR + ); + }); + describe('STORAGE_EMULATOR_HOST', () => { // Note: EMULATOR_HOST is an experimental configuration variable. Use apiEndpoint instead. const EMULATOR_HOST = 'https://internal.benchmark.com/path';