From 16a741e88c72b9bad43e211cf72ee21f869f8c8e Mon Sep 17 00:00:00 2001 From: TheEdward162 Date: Fri, 15 Mar 2024 16:12:19 +0100 Subject: [PATCH 1/2] feat: improve map-std to allow handling multipart bodies * add `HttpResponse.bodyBuffer()` * add `Buffer.concat()` * add `Buffer.length` * add new encoding `base64url` which uses urlsafe alphabet --- .../src/core_to_map_bindings/mod.rs | 1 + .../src/core_to_map_bindings/unstable.rs | 17 +++++++--- core_js/core-ffi/ffi.d.ts | 4 +-- core_js/map-std/src/internal/bytes.ts | 33 ++++++++++++------- core_js/map-std/src/internal/message.ts | 2 +- .../{node_compat.ts => node_buffer.ts} | 29 ++++++++++++++-- core_js/map-std/src/internal/types.ts | 2 +- core_js/map-std/src/map_std.ts | 2 +- core_js/map-std/src/unstable.ts | 11 ++++--- 9 files changed, 73 insertions(+), 28 deletions(-) rename core_js/map-std/src/internal/{node_compat.ts => node_buffer.ts} (60%) diff --git a/core/interpreter_js/src/core_to_map_bindings/mod.rs b/core/interpreter_js/src/core_to_map_bindings/mod.rs index 28715f30..159d1735 100644 --- a/core/interpreter_js/src/core_to_map_bindings/mod.rs +++ b/core/interpreter_js/src/core_to_map_bindings/mod.rs @@ -74,6 +74,7 @@ macro_rules! ensure_arguments { ) }; + (#arg_type(bool) $val: expr) => { $val.as_bool().ok() }; (#arg_type(i32) $val: expr) => { $val.try_as_integer().ok() }; (#arg_type(str) $val: expr) => { $val.as_str().ok() }; (#arg_type(mut_bytes) $val: expr) => { $val.as_bytes_mut().ok() }; diff --git a/core/interpreter_js/src/core_to_map_bindings/unstable.rs b/core/interpreter_js/src/core_to_map_bindings/unstable.rs index a931a998..6f06455b 100644 --- a/core/interpreter_js/src/core_to_map_bindings/unstable.rs +++ b/core/interpreter_js/src/core_to_map_bindings/unstable.rs @@ -25,7 +25,7 @@ pub fn link( // debug "printDebug": __export_print_debug, "print": __export_print, - // env + // coding "bytes_to_utf8": __export_bytes_to_utf8, "utf8_to_bytes": __export_utf8_to_bytes, "bytes_to_base64": __export_bytes_to_base64, @@ -146,14 +146,21 @@ fn __export_utf8_to_bytes<'ctx, H: MapStdUnstable + 'static>( Ok(JSValue::ArrayBuffer(string.into())) } +fn get_base64_alphabet(url_safe: bool, pad: bool) -> base64::engine::GeneralPurpose { + match (url_safe, pad) { + (false, true) => base64::engine::general_purpose::STANDARD, + (true, true) => base64::engine::general_purpose::URL_SAFE, + _ => unimplemented!() + } +} fn __export_bytes_to_base64<'ctx, H: MapStdUnstable + 'static>( _state: &mut H, _this: &JSValueRef<'ctx>, args: &[JSValueRef<'ctx>], ) -> Result { - let bytes = ensure_arguments!("bytes_to_base64" args; 0: bytes); + let (bytes, url_safe) = ensure_arguments!("bytes_to_base64" args; 0: bytes, 1: bool); - let result = base64::engine::general_purpose::STANDARD.encode(bytes); + let result = get_base64_alphabet(url_safe, true).encode(bytes); Ok(result.into()) } @@ -163,9 +170,9 @@ fn __export_base64_to_bytes<'ctx, H: MapStdUnstable + 'static>( _this: &JSValueRef<'ctx>, args: &[JSValueRef<'ctx>], ) -> Result { - let string = ensure_arguments!("base64_to_bytes" args; 0: str); + let (string, url_safe) = ensure_arguments!("base64_to_bytes" args; 0: str, 1: bool); - match base64::engine::general_purpose::STANDARD.decode(string) { + match get_base64_alphabet(url_safe, true).decode(string) { Err(err) => Err(JSError::Type(format!( "Could not decode string as base64: {}", err diff --git a/core_js/core-ffi/ffi.d.ts b/core_js/core-ffi/ffi.d.ts index 2636b57a..37271ad3 100644 --- a/core_js/core-ffi/ffi.d.ts +++ b/core_js/core-ffi/ffi.d.ts @@ -10,8 +10,8 @@ declare type StdFfi = { // coding bytes_to_utf8(bytes: ArrayBuffer): string, utf8_to_bytes(utf8: string): ArrayBuffer, - bytes_to_base64(bytes: ArrayBuffer): string, - base64_to_bytes(base64: string): ArrayBuffer, + bytes_to_base64(bytes: ArrayBuffer, url_safe: boolean): string, + base64_to_bytes(base64: string, url_safe: boolean): ArrayBuffer, record_to_urlencoded(value: Record): string, // env print(message: string): void, diff --git a/core_js/map-std/src/internal/bytes.ts b/core_js/map-std/src/internal/bytes.ts index 37fd12ad..cb72b0f5 100644 --- a/core_js/map-std/src/internal/bytes.ts +++ b/core_js/map-std/src/internal/bytes.ts @@ -70,23 +70,32 @@ export class Bytes { // TODO: again support for TypedArrays in Javy const buffer = this.#buffer.buffer.slice(0, this.len); - if (encoding === 'utf8') { - return __ffi.unstable.bytes_to_utf8(buffer); - } else if (encoding === 'base64') { - return __ffi.unstable.bytes_to_base64(buffer); + switch (encoding) { + case 'utf8': + return __ffi.unstable.bytes_to_utf8(buffer); + case 'base64': + return __ffi.unstable.bytes_to_base64(buffer, false); + case 'base64url': + return __ffi.unstable.bytes_to_base64(buffer, true); + default: + throw new Error(`encoding "${encoding}" not implemented`); } - - throw new Error(`encoding "${encoding}" not implemented`); } static encode(string: string, encoding: Encoding = 'utf8'): Bytes { let buffer; - if (encoding === 'utf8') { - buffer = __ffi.unstable.utf8_to_bytes(string); - } else if (encoding === 'base64') { - buffer = __ffi.unstable.base64_to_bytes(string); - } else { - throw new Error(`encoding "${encoding}" not implemented`); + switch (encoding) { + case 'utf8': + buffer = __ffi.unstable.utf8_to_bytes(string); + break; + case 'base64': + buffer = __ffi.unstable.base64_to_bytes(string, false); + break; + case 'base64url': + buffer = __ffi.unstable.base64_to_bytes(string, true); + break; + default: + throw new Error(`encoding "${encoding}" not implemented`); } return new Bytes(new Uint8Array(buffer), buffer.byteLength); diff --git a/core_js/map-std/src/internal/message.ts b/core_js/map-std/src/internal/message.ts index 92ec4e43..6154d475 100644 --- a/core_js/map-std/src/internal/message.ts +++ b/core_js/map-std/src/internal/message.ts @@ -1,4 +1,4 @@ -import { Buffer } from './node_compat'; +import { Buffer } from './node_buffer'; export function jsonReplacerMapValue(key: any, value: any): any { // TODO: this is how node Buffer gets serialized - do we want that? diff --git a/core_js/map-std/src/internal/node_compat.ts b/core_js/map-std/src/internal/node_buffer.ts similarity index 60% rename from core_js/map-std/src/internal/node_compat.ts rename to core_js/map-std/src/internal/node_buffer.ts index 56c3f02e..d98ff6fe 100644 --- a/core_js/map-std/src/internal/node_compat.ts +++ b/core_js/map-std/src/internal/node_buffer.ts @@ -1,7 +1,6 @@ import { Bytes } from './bytes'; import type { Encoding } from './types'; -// TODO: Comlink/node map compat export class Buffer { static from(value: unknown, encoding: Encoding = 'utf8'): Buffer { if (typeof value === 'string') { @@ -31,6 +30,28 @@ export class Buffer { return value instanceof Buffer; } + static concat(list: Buffer[], totalLength?: number): Buffer { + if (totalLength == undefined) { + totalLength = list.reduce((acc, curr) => acc + curr.length, 0) + } + + const bytes = Bytes.withCapacity(totalLength) + for (const buf of list) { + if (bytes.len === bytes.capacity) { + break; + } + + const remaining = bytes.capacity - bytes.len; + if (buf.length <= remaining) { + bytes.extend(buf.inner.data); + } else { + bytes.extend(buf.inner.data.slice(0, remaining)); + } + } + + return new Buffer(bytes) + } + #inner: Bytes; private constructor(inner: Bytes) { this.#inner = inner; @@ -41,7 +62,11 @@ export class Buffer { return this.#inner; } + get length(): number { + return this.#inner.len; + } + public toString(encoding: Encoding = 'utf8'): string { return this.#inner.decode(encoding); } -} \ No newline at end of file +} diff --git a/core_js/map-std/src/internal/types.ts b/core_js/map-std/src/internal/types.ts index 44daf064..b08abf4e 100644 --- a/core_js/map-std/src/internal/types.ts +++ b/core_js/map-std/src/internal/types.ts @@ -1,2 +1,2 @@ export type MultiMap = Record; -export type Encoding = 'utf8' | 'base64'; +export type Encoding = 'utf8' | 'base64' | 'base64url'; diff --git a/core_js/map-std/src/map_std.ts b/core_js/map-std/src/map_std.ts index dd9427e8..4e03a7ff 100644 --- a/core_js/map-std/src/map_std.ts +++ b/core_js/map-std/src/map_std.ts @@ -1,4 +1,4 @@ -import { Buffer as NodeBuffer } from './internal/node_compat'; +import { Buffer as NodeBuffer } from './internal/node_buffer'; import * as unstable from './unstable'; declare global { diff --git a/core_js/map-std/src/unstable.ts b/core_js/map-std/src/unstable.ts index 206948e7..d294a6de 100644 --- a/core_js/map-std/src/unstable.ts +++ b/core_js/map-std/src/unstable.ts @@ -2,7 +2,7 @@ import { messageExchange, jsonReviverMapValue, jsonReplacerMapValue, responseErr import { ensureMultimap } from './internal/util'; import { Bytes, ByteStream } from './internal/bytes'; import type { MultiMap } from './internal/types'; -import { Buffer } from './internal/node_compat'; +import { Buffer } from './internal/node_buffer'; export type { MultiMap, Encoding } from './internal/types'; @@ -53,12 +53,15 @@ export class HttpResponse { this.#bodyStream = new ByteStream(bodyStream); } - // TODO: either make Bytes public or use a different type private bodyBytes(): Bytes { - const buffer = this.#bodyStream.readToEnd(); + const bytes = this.#bodyStream.readToEnd(); this.#bodyStream.close(); - return buffer; + return bytes; + } + + public bodyBuffer(): Buffer { + return Buffer.from(this.bodyBytes()); } public bodyText(): string { From 5f5f9e4c410ab5263bd92e861e91f42c944d01b4 Mon Sep 17 00:00:00 2001 From: TheEdward162 Date: Mon, 18 Mar 2024 10:22:01 +0100 Subject: [PATCH 2/2] fix: use TypedArray.subarray instead of .slice in Buffer.concat --- core_js/map-std/src/internal/node_buffer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core_js/map-std/src/internal/node_buffer.ts b/core_js/map-std/src/internal/node_buffer.ts index d98ff6fe..24e4e024 100644 --- a/core_js/map-std/src/internal/node_buffer.ts +++ b/core_js/map-std/src/internal/node_buffer.ts @@ -45,7 +45,7 @@ export class Buffer { if (buf.length <= remaining) { bytes.extend(buf.inner.data); } else { - bytes.extend(buf.inner.data.slice(0, remaining)); + bytes.extend(buf.inner.data.subarray(0, remaining)); } }