From 3f23a8048540f558765db1bbb6af4759af3ef7f8 Mon Sep 17 00:00:00 2001 From: Nicholas Paun Date: Wed, 4 Sep 2024 20:03:13 -0700 Subject: [PATCH] Implement brotli one shot methods --- src/node/internal/internal_zlib.ts | 38 ++++ src/node/internal/zlib.d.ts | 20 ++ src/node/zlib.ts | 12 ++ .../api/node/tests/zlib-nodejs-test.js | 57 +++++- src/workerd/api/node/zlib-util.c++ | 185 ++++++++++++------ src/workerd/api/node/zlib-util.h | 32 ++- 6 files changed, 281 insertions(+), 63 deletions(-) diff --git a/src/node/internal/internal_zlib.ts b/src/node/internal/internal_zlib.ts index 47d5142df9a..2bed292fde5 100644 --- a/src/node/internal/internal_zlib.ts +++ b/src/node/internal/internal_zlib.ts @@ -87,6 +87,20 @@ export function unzipSync( return Buffer.from(zlibUtil.zlibSync(data, options, zlibUtil.CONST_UNZIP)); } +export function brotliDecompressSync( + data: ArrayBufferView | string, + options: BrotliOptions = {} +): Buffer { + return Buffer.from(zlibUtil.brotliDecompressSync(data, options)); +} + +export function brotliCompressSync( + data: ArrayBufferView | string, + options: BrotliOptions = {} +): Buffer { + return Buffer.from(zlibUtil.brotliCompressSync(data, options)); +} + function normalizeArgs( optionsOrCallback: ZlibOptions | CompressCallback, callbackOrUndefined?: CompressCallback @@ -209,6 +223,30 @@ export function gzip( zlibUtil.zlib(data, options, zlibUtil.CONST_GZIP, wrapCallback(callback)); } +export function brotliDecompress( + data: ArrayBufferView | string, + optionsOrCallback: BrotliOptions | CompressCallback, + callbackOrUndefined?: CompressCallback +): void { + const [options, callback] = normalizeArgs( + optionsOrCallback, + callbackOrUndefined + ); + zlibUtil.brotliDecompress(data, options, wrapCallback(callback)); +} + +export function brotliCompress( + data: ArrayBufferView | string, + optionsOrCallback: BrotliOptions | CompressCallback, + callbackOrUndefined?: CompressCallback +): void { + const [options, callback] = normalizeArgs( + optionsOrCallback, + callbackOrUndefined + ); + zlibUtil.brotliCompress(data, options, wrapCallback(callback)); +} + export class Gzip extends Zlib { public constructor(options: ZlibOptions) { super(options, CONST_GZIP); diff --git a/src/node/internal/zlib.d.ts b/src/node/internal/zlib.d.ts index f46e711bb84..8b4017ca4eb 100644 --- a/src/node/internal/zlib.d.ts +++ b/src/node/internal/zlib.d.ts @@ -18,7 +18,27 @@ export function zlib( options: ZlibOptions, mode: number, cb: CompressCallback +): void; + +export function brotliDecompressSync( + data: ArrayBufferView | string, + options: BrotliOptions ): ArrayBuffer; +export function brotliDecompress( + data: ArrayBufferView | string, + options: BrotliOptions, + cb: CompressCallback +): void; + +export function brotliCompressSync( + data: ArrayBufferView | string, + options: BrotliOptions +): ArrayBuffer; +export function brotliCompress( + data: ArrayBufferView | string, + options: BrotliOptions, + cb: CompressCallback +): void; // zlib.constants (part of the API contract for node:zlib) export const CONST_Z_NO_FLUSH: number; diff --git a/src/node/zlib.ts b/src/node/zlib.ts index 6c45ff7220a..93bb9867ad0 100644 --- a/src/node/zlib.ts +++ b/src/node/zlib.ts @@ -49,6 +49,10 @@ const gunzip = protectMethod(zlib.gunzip); const gunzipSync = protectMethod(zlib.gunzipSync); const unzip = protectMethod(zlib.unzip); const unzipSync = protectMethod(zlib.unzipSync); +const brotliCompress = protectMethod(zlib.brotliCompress); +const brotliDecompressSync = protectMethod(zlib.brotliDecompressSync); +const brotliDecompress = protectMethod(zlib.brotliDecompress); +const brotliCompressSync = protectMethod(zlib.brotliCompressSync); export { crc32, @@ -92,6 +96,10 @@ export { gunzipSync, unzip, unzipSync, + brotliDecompress, + brotliDecompressSync, + brotliCompress, + brotliCompressSync, }; export default { @@ -136,4 +144,8 @@ export default { gunzipSync, unzip, unzipSync, + brotliDecompress, + brotliDecompressSync, + brotliCompress, + brotliCompressSync, }; diff --git a/src/workerd/api/node/tests/zlib-nodejs-test.js b/src/workerd/api/node/tests/zlib-nodejs-test.js index 80c083a6c81..545a0091043 100644 --- a/src/workerd/api/node/tests/zlib-nodejs-test.js +++ b/src/workerd/api/node/tests/zlib-nodejs-test.js @@ -2090,6 +2090,61 @@ export const convenienceMethods = { }, }; +// Test taken from +// https://github.com/nodejs/node/blob/2bd6a57b7b934afcaf437a90e2abebcb79c13acf/test/parallel/test-zlib-brotli-from-string.js +export const brotliFromString = { + async test() { + const inputString = + 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' + + 't. Morbi faucibus, purus at gravida dictum, libero arcu ' + + 'convallis lacus, in commodo libero metus eu nisi. Nullam' + + ' commodo, neque nec porta placerat, nisi est fermentum a' + + 'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' + + 'n non diam orci. Proin quis elit turpis. Suspendisse non' + + ' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' + + 'm arcu mi, sodales non suscipit id, ultrices ut massa. S' + + 'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. '; + const compressedString = + 'G/gBQBwHdky2aHV5KK9Snf05//1pPdmNw/7232fnIm1IB' + + 'K1AA8RsN8OB8Nb7Lpgk3UWWUlzQXZyHQeBBbXMTQXC1j7' + + 'wg3LJs9LqOGHRH2bj/a2iCTLLx8hBOyTqgoVuD1e+Qqdn' + + 'f1rkUNyrWq6LtOhWgxP3QUwdhKGdZm3rJWaDDBV7+pDk1' + + 'MIkrmjp4ma2xVi5MsgJScA3tP1I7mXeby6MELozrwoBQD' + + 'mVTnEAicZNj4lkGqntJe2qSnGyeMmcFgraK94vCg/4iLu' + + 'Tw5RhKhnVY++dZ6niUBmRqIutsjf5TzwF5iAg8a9UkjF5' + + '2eZ0tB2vo6v8SqVfNMkBmmhxr0NT9LkYF69aEjlYzj7IE' + + 'KmEUQf1HBogRYhFIt4ymRNEgHAIzOyNEsQM='; + + { + const { promise, resolve } = Promise.withResolvers(); + + zlib.brotliCompress(inputString, (err, buffer) => { + assert.ifError(err); + assert(inputString.length > buffer.length); + + zlib.brotliDecompress(buffer, (err, buffer) => { + assert.ifError(err); + assert.strictEqual(buffer.toString(), inputString); + resolve(); + }); + }); + await promise; + } + + { + const { promise, resolve } = Promise.withResolvers(); + const buffer = Buffer.from(compressedString, 'base64'); + zlib.brotliDecompress(buffer, (err, buffer) => { + assert.ifError(err); + assert.strictEqual(buffer.toString(), inputString); + resolve(); + }); + + await promise; + } + }, +}; + // Node.js tests relevant to zlib // // - [ ] test-zlib-brotli-16GB.js @@ -2107,7 +2162,7 @@ export const convenienceMethods = { // - [x] test-zlib-flush-flags.js // - [ ] test-zlib-kmaxlength-rangeerror.js // - [ ] test-zlib-unused-weak.js -// - [ ] test-zlib-brotli-from-string.js +// - [x] test-zlib-brotli-from-string.js // - [x] test-zlib-deflate-constructors.js // - [x] test-zlib-flush.js // - [ ] test-zlib-maxOutputLength.js diff --git a/src/workerd/api/node/zlib-util.c++ b/src/workerd/api/node/zlib-util.c++ index 79de5136372..0b4b05a37c4 100644 --- a/src/workerd/api/node/zlib-util.c++ +++ b/src/workerd/api/node/zlib-util.c++ @@ -4,6 +4,7 @@ // Copyright Joyent and Node contributors. All rights reserved. MIT license. #include "util.h" +#include #include "zlib-util.h" #include "nbytes.h" @@ -13,8 +14,7 @@ // Latest implementation of Node.js zlib can be found at: // https://github.com/nodejs/node/blob/main/src/node_zlib.cc namespace workerd::api::node { - -kj::ArrayPtr ZlibUtil::getInputFromSource(InputSource& data) { +kj::ArrayPtr getInputFromSource(ZlibUtil::InputSource& data) { KJ_SWITCH_ONEOF(data) { KJ_CASE_ONEOF(dataBuf, kj::Array) { JSG_REQUIRE(dataBuf.size() < Z_MAX_CHUNK, RangeError, "Memory limit exceeded"_kj); @@ -624,6 +624,20 @@ void BrotliContext::setBuffers(kj::ArrayPtr input, availOut = outputLength; } +void BrotliContext::setInputBuffer(kj::ArrayPtr input) { + nextIn = input.begin(); + availIn = input.size(); +} + +void BrotliContext::setOutputBuffer(kj::ArrayPtr output) { + nextOut = output.begin(); + availOut = output.size(); +} + +kj::uint BrotliContext::getAvailOut() const { + return availOut; +} + void BrotliContext::setFlush(int _flush) { flush = static_cast(_flush); } @@ -691,62 +705,6 @@ kj::Maybe BrotliEncoderContext::getError() const { return kj::none; } -kj::Array syncProcessBuffer(ZlibContext& ctx, GrowableBuffer& result) { - do { - result.addChunk(); - ctx.setOutputBuffer(kj::ArrayPtr(result.end(), result.available())); - - ctx.work(); - - KJ_IF_SOME(error, ctx.getError()) { - JSG_FAIL_REQUIRE(Error, error.message); - } - - result.adjustUnused(ctx.getAvailOut()); - } while (ctx.getAvailOut() == 0); - - return result.releaseAsArray(); -} - -// It's ZlibContext but it's RAII -class ZlibContextRAII: public ZlibContext { -public: - using ZlibContext::ZlibContext; - - ~ZlibContextRAII() { - close(); - } -}; - -kj::Array ZlibUtil::zlibSync(InputSource data, Options opts, ZlibModeValue mode) { - ZlibContextRAII ctx; - - auto chunkSize = opts.chunkSize.orDefault(ZLIB_PERFORMANT_CHUNK_SIZE); - auto maxOutputLength = opts.maxOutputLength.orDefault(Z_MAX_CHUNK); - - JSG_REQUIRE(Z_MIN_CHUNK <= chunkSize && chunkSize <= Z_MAX_CHUNK, Error, "Invalid chunkSize"); - JSG_REQUIRE(maxOutputLength <= Z_MAX_CHUNK, Error, "Invalid maxOutputLength"); - GrowableBuffer result(ZLIB_PERFORMANT_CHUNK_SIZE, maxOutputLength); - - ctx.setMode(static_cast(mode)); - ctx.initialize(opts.level.orDefault(Z_DEFAULT_LEVEL), - opts.windowBits.orDefault(Z_DEFAULT_WINDOWBITS), opts.memLevel.orDefault(Z_DEFAULT_MEMLEVEL), - opts.strategy.orDefault(Z_DEFAULT_STRATEGY), kj::mv(opts.dictionary)); - ctx.setFlush(opts.finishFlush.orDefault(Z_FINISH)); - ctx.setInputBuffer(getInputFromSource(data)); - return syncProcessBuffer(ctx, result); -} - -void ZlibUtil::zlibWithCallback( - jsg::Lock& js, InputSource data, Options options, ZlibModeValue mode, CompressCallback cb) { - try { - cb(js, kj::none, zlibSync(kj::mv(data), kj::mv(options), mode)); - } catch (kj::Exception& ex) { - auto tunneledError = jsg::tunneledErrorType(ex.getDescription()); - cb(js, tunneledError.message, kj::none); - } -} - BrotliDecoderContext::BrotliDecoderContext(ZlibMode _mode): BrotliContext(_mode) { auto instance = BrotliDecoderCreateInstance(alloc_brotli, free_brotli, alloc_opaque_brotli); state = kj::disposeWith(instance); @@ -878,6 +836,109 @@ void ZlibUtil::CompressionStream::FreeForZlib(void* data, vo auto real_pointer = static_cast(pointer) - sizeof(size_t); JSG_REQUIRE(ctx->allocations.erase(real_pointer), Error, "Zlib allocation should exist"_kj); } +namespace { +// A RAII wrapper around a compression context class +template +class ContextRAII: public CompressionContext { +public: + using CompressionContext::CompressionContext; + + ~ContextRAII() { + static_cast(this)->close(); + } +}; + +template +static kj::Array syncProcessBuffer(Context& ctx, GrowableBuffer& result) { + do { + result.addChunk(); + ctx.setOutputBuffer(kj::ArrayPtr(result.end(), result.available())); + + ctx.work(); + + KJ_IF_SOME(error, ctx.getError()) { + JSG_FAIL_REQUIRE(Error, error.message); + } + + result.adjustUnused(ctx.getAvailOut()); + } while (ctx.getAvailOut() == 0); + + return result.releaseAsArray(); +} +} // namespace + +kj::Array ZlibUtil::zlibSync( + ZlibUtil::InputSource data, ZlibUtil::Options opts, ZlibModeValue mode) { + ContextRAII ctx; + + auto chunkSize = opts.chunkSize.orDefault(ZLIB_PERFORMANT_CHUNK_SIZE); + auto maxOutputLength = opts.maxOutputLength.orDefault(Z_MAX_CHUNK); + + JSG_REQUIRE(Z_MIN_CHUNK <= chunkSize && chunkSize <= Z_MAX_CHUNK, Error, "Invalid chunkSize"); + JSG_REQUIRE(maxOutputLength <= Z_MAX_CHUNK, Error, "Invalid maxOutputLength"); + GrowableBuffer result(ZLIB_PERFORMANT_CHUNK_SIZE, maxOutputLength); + + ctx.setMode(static_cast(mode)); + ctx.initialize(opts.level.orDefault(Z_DEFAULT_LEVEL), + opts.windowBits.orDefault(Z_DEFAULT_WINDOWBITS), opts.memLevel.orDefault(Z_DEFAULT_MEMLEVEL), + opts.strategy.orDefault(Z_DEFAULT_STRATEGY), kj::mv(opts.dictionary)); + ctx.setFlush(opts.finishFlush.orDefault(Z_FINISH)); + ctx.setInputBuffer(getInputFromSource(data)); + return syncProcessBuffer(ctx, result); +} + +void ZlibUtil::zlibWithCallback( + jsg::Lock& js, InputSource data, Options options, ZlibModeValue mode, CompressCallback cb) { + try { + cb(js, kj::none, zlibSync(kj::mv(data), kj::mv(options), mode)); + } catch (kj::Exception& ex) { + auto tunneledError = jsg::tunneledErrorType(ex.getDescription()); + cb(js, tunneledError.message, kj::none); + } +} + +template +kj::Array ZlibUtil::brotliSync(InputSource data, BrotliOptions opts) { + ContextRAII ctx(Context::Mode); + + auto chunkSize = opts.chunkSize.orDefault(ZLIB_PERFORMANT_CHUNK_SIZE); + auto maxOutputLength = opts.maxOutputLength.orDefault(Z_MAX_CHUNK); + + JSG_REQUIRE(Z_MIN_CHUNK <= chunkSize && chunkSize <= Z_MAX_CHUNK, Error, "Invalid chunkSize"); + JSG_REQUIRE(maxOutputLength <= Z_MAX_CHUNK, Error, "Invalid maxOutputLength"); + GrowableBuffer result(ZLIB_PERFORMANT_CHUNK_SIZE, maxOutputLength); + + // TODO(soon): should we track them brotli allocationz? + auto maybeError = ctx.initialize(nullptr, nullptr, nullptr); + + KJ_IF_SOME(err, maybeError) { + JSG_FAIL_REQUIRE(Error, err.message); + } + + KJ_IF_SOME(params, opts.params) { + for (const auto& field: params.fields) { + maybeError = ctx.setParams(field.name.parseAs(), field.value); + KJ_IF_SOME(err, maybeError) { + JSG_FAIL_REQUIRE(Error, err.message); + } + } + } + + ctx.setFlush(opts.finishFlush.orDefault(BROTLI_OPERATION_FINISH)); + ctx.setInputBuffer(getInputFromSource(data)); + return syncProcessBuffer(ctx, result); +} + +template +void ZlibUtil::brotliWithCallback( + jsg::Lock& js, InputSource data, BrotliOptions options, CompressCallback cb) { + try { + cb(js, kj::none, brotliSync(kj::mv(data), kj::mv(options))); + } catch (kj::Exception& ex) { + auto tunneledError = jsg::tunneledErrorType(ex.getDescription()); + cb(js, tunneledError.message, kj::none); + } +} #ifndef CREATE_TEMPLATE #define CREATE_TEMPLATE(T) \ @@ -907,6 +968,14 @@ template bool ZlibUtil::BrotliCompressionStream::initializ template bool ZlibUtil::BrotliCompressionStream::initialize( jsg::Lock&, jsg::BufferSource, jsg::BufferSource, jsg::Function); +template kj::Array ZlibUtil::brotliSync( + InputSource data, BrotliOptions opts); +template kj::Array ZlibUtil::brotliSync( + InputSource data, BrotliOptions opts); +template void ZlibUtil::brotliWithCallback( + jsg::Lock& js, InputSource data, BrotliOptions options, CompressCallback cb); +template void ZlibUtil::brotliWithCallback( + jsg::Lock& js, InputSource data, BrotliOptions options, CompressCallback cb); #undef CREATE_TEMPLATE #endif } // namespace workerd::api::node diff --git a/src/workerd/api/node/zlib-util.h b/src/workerd/api/node/zlib-util.h index e6b78cba2c2..c7e462d0299 100644 --- a/src/workerd/api/node/zlib-util.h +++ b/src/workerd/api/node/zlib-util.h @@ -190,7 +190,10 @@ class BrotliContext { uint32_t inputLength, kj::ArrayPtr output, uint32_t outputLength); + void setInputBuffer(kj::ArrayPtr input); + void setOutputBuffer(kj::ArrayPtr output); void setFlush(int flush); + kj::uint getAvailOut() const; void getAfterWriteResult(uint32_t* availIn, uint32_t* availOut) const; void setMode(ZlibMode _mode) { mode = _mode; @@ -212,8 +215,9 @@ class BrotliContext { void* alloc_opaque_brotli = nullptr; }; -class BrotliEncoderContext final: public BrotliContext { +class BrotliEncoderContext: public BrotliContext { public: + static const ZlibMode Mode = ZlibMode::BROTLI_ENCODE; explicit BrotliEncoderContext(ZlibMode _mode); void close(); @@ -230,8 +234,9 @@ class BrotliEncoderContext final: public BrotliContext { kj::Own state; }; -class BrotliDecoderContext final: public BrotliContext { +class BrotliDecoderContext: public BrotliContext { public: + static const ZlibMode Mode = ZlibMode::BROTLI_DECODE; explicit BrotliDecoderContext(ZlibMode _mode); void close(); @@ -411,21 +416,40 @@ class ZlibUtil final: public jsg::Object { maxOutputLength); }; + struct BrotliOptions { + jsg::Optional flush; + jsg::Optional finishFlush; + jsg::Optional chunkSize; + jsg::Optional> params; + jsg::Optional maxOutputLength; + JSG_STRUCT(flush, finishFlush, chunkSize, params, maxOutputLength); + }; + using InputSource = kj::OneOf, kj::Array>; using CompressCallback = jsg::Function, jsg::Optional>)>; - static kj::ArrayPtr getInputFromSource(InputSource& data); uint32_t crc32Sync(InputSource data, uint32_t value); void zlibWithCallback( jsg::Lock& js, InputSource data, Options options, ZlibModeValue mode, CompressCallback cb); kj::Array zlibSync(InputSource data, Options options, ZlibModeValue mode); + template + kj::Array brotliSync(InputSource data, BrotliOptions options); + template + void brotliWithCallback( + jsg::Lock& js, InputSource data, BrotliOptions options, CompressCallback cb); + JSG_RESOURCE_TYPE(ZlibUtil) { JSG_METHOD_NAMED(crc32, crc32Sync); JSG_METHOD(zlibSync); JSG_METHOD_NAMED(zlib, zlibWithCallback); + JSG_METHOD_NAMED(brotliDecompressSync, template brotliSync); + JSG_METHOD_NAMED(brotliCompressSync, template brotliSync); + JSG_METHOD_NAMED(brotliDecompress, template brotliWithCallback); + JSG_METHOD_NAMED(brotliCompress, template brotliWithCallback); + JSG_NESTED_TYPE(ZlibStream); JSG_NESTED_TYPE_NAMED(BrotliCompressionStream, BrotliEncoder); JSG_NESTED_TYPE_NAMED(BrotliCompressionStream, BrotliDecoder); @@ -587,7 +611,7 @@ class ZlibUtil final: public jsg::Object { api::node::ZlibUtil::CompressionStream, \ api::node::ZlibUtil::CompressionStream, \ api::node::ZlibUtil::CompressionStream, \ - api::node::ZlibUtil::Options + api::node::ZlibUtil::Options, api::node::ZlibUtil::BrotliOptions } // namespace workerd::api::node