feat: improve map-std to allow handling multipart bodies

* add `HttpResponse.bodyBuffer()`
* add `Buffer.concat()`
* add `Buffer.length`
* add new encoding `base64url` which uses urlsafe alphabet

core/interpreter_js/src/core_to_map_bindings/mod.rs

@@ -74,6 +74,7 @@ macro_rules! ensure_arguments {
         )
     };
 
+    (#arg_type(bool) $val: expr) => { $val.as_bool().ok() };
     (#arg_type(i32) $val: expr) => { $val.try_as_integer().ok() };
     (#arg_type(str) $val: expr) => { $val.as_str().ok() };
     (#arg_type(mut_bytes) $val: expr) => { $val.as_bytes_mut().ok() };

core/interpreter_js/src/core_to_map_bindings/unstable.rs

@@ -25,7 +25,7 @@ pub fn link<H: MapStdUnstable + 'static>(
             // debug
             "printDebug": __export_print_debug,
             "print": __export_print,
-            // env
+            // coding
             "bytes_to_utf8": __export_bytes_to_utf8,
             "utf8_to_bytes": __export_utf8_to_bytes,
             "bytes_to_base64": __export_bytes_to_base64,
@@ -146,14 +146,21 @@ fn __export_utf8_to_bytes<'ctx, H: MapStdUnstable + 'static>(
     Ok(JSValue::ArrayBuffer(string.into()))
 }
 
+fn get_base64_alphabet(url_safe: bool, pad: bool) -> base64::engine::GeneralPurpose {
+    match (url_safe, pad) {
+        (false, true) => base64::engine::general_purpose::STANDARD,
+        (true, true) => base64::engine::general_purpose::URL_SAFE,
+        _ => unimplemented!()
+    }
+}
 fn __export_bytes_to_base64<'ctx, H: MapStdUnstable + 'static>(
     _state: &mut H,
     _this: &JSValueRef<'ctx>,
     args: &[JSValueRef<'ctx>],
 ) -> Result<JSValue, JSError> {
-    let bytes = ensure_arguments!("bytes_to_base64" args; 0: bytes);
+    let (bytes, url_safe) = ensure_arguments!("bytes_to_base64" args; 0: bytes, 1: bool);
 
-    let result = base64::engine::general_purpose::STANDARD.encode(bytes);
+    let result = get_base64_alphabet(url_safe, true).encode(bytes);
 
     Ok(result.into())
 }
@@ -163,9 +170,9 @@ fn __export_base64_to_bytes<'ctx, H: MapStdUnstable + 'static>(
     _this: &JSValueRef<'ctx>,
     args: &[JSValueRef<'ctx>],
 ) -> Result<JSValue, JSError> {
-    let string = ensure_arguments!("base64_to_bytes" args; 0: str);
+    let (string, url_safe) = ensure_arguments!("base64_to_bytes" args; 0: str, 1: bool);
 
-    match base64::engine::general_purpose::STANDARD.decode(string) {
+    match get_base64_alphabet(url_safe, true).decode(string) {
         Err(err) => Err(JSError::Type(format!(
             "Could not decode string as base64: {}",
             err

core_js/core-ffi/ffi.d.ts

@@ -10,8 +10,8 @@ declare type StdFfi = {
     // coding
     bytes_to_utf8(bytes: ArrayBuffer): string,
     utf8_to_bytes(utf8: string): ArrayBuffer,
-    bytes_to_base64(bytes: ArrayBuffer): string,
-    base64_to_bytes(base64: string): ArrayBuffer,
+    bytes_to_base64(bytes: ArrayBuffer, url_safe: boolean): string,
+    base64_to_bytes(base64: string, url_safe: boolean): ArrayBuffer,
     record_to_urlencoded(value: Record<string, string[]>): string,
     // env
     print(message: string): void,

core_js/map-std/src/internal/bytes.ts

@@ -70,23 +70,32 @@ export class Bytes {
     // TODO: again support for TypedArrays in Javy
     const buffer = this.#buffer.buffer.slice(0, this.len);
 
-    if (encoding === 'utf8') {
-      return __ffi.unstable.bytes_to_utf8(buffer);
-    } else if (encoding === 'base64') {
-      return __ffi.unstable.bytes_to_base64(buffer);
+    switch (encoding) {
+      case 'utf8':
+        return __ffi.unstable.bytes_to_utf8(buffer);
+      case 'base64':
+        return __ffi.unstable.bytes_to_base64(buffer, false);
+      case 'base64url':
+        return __ffi.unstable.bytes_to_base64(buffer, true);
+      default:
+        throw new Error(`encoding "${encoding}" not implemented`);
     }
-
-    throw new Error(`encoding "${encoding}" not implemented`);
   }
 
   static encode(string: string, encoding: Encoding = 'utf8'): Bytes {
     let buffer;
-    if (encoding === 'utf8') {
-      buffer = __ffi.unstable.utf8_to_bytes(string);
-    } else if (encoding === 'base64') {
-      buffer = __ffi.unstable.base64_to_bytes(string);
-    } else {
-      throw new Error(`encoding "${encoding}" not implemented`);
+    switch (encoding) {
+      case 'utf8':
+        buffer = __ffi.unstable.utf8_to_bytes(string);
+        break;
+      case 'base64':
+        buffer = __ffi.unstable.base64_to_bytes(string, false);
+        break;
+      case 'base64url':
+        buffer = __ffi.unstable.base64_to_bytes(string, true);
+        break;
+      default:
+        throw new Error(`encoding "${encoding}" not implemented`);
     }
 
     return new Bytes(new Uint8Array(buffer), buffer.byteLength);

core_js/map-std/src/internal/message.ts

@@ -1,4 +1,4 @@
-import { Buffer } from './node_compat';
+import { Buffer } from './node_buffer';
 
 export function jsonReplacerMapValue(key: any, value: any): any {
   // TODO: this is how node Buffer gets serialized - do we want that?

core_js/map-std/src/internal/node_compat.ts → core_js/map-std/src/internal/node_buffer.ts

@@ -1,7 +1,6 @@
 import { Bytes } from './bytes';
 import type { Encoding } from './types';
 
-// TODO: Comlink/node map compat
 export class Buffer {
   static from(value: unknown, encoding: Encoding = 'utf8'): Buffer {
     if (typeof value === 'string') {
@@ -31,6 +30,28 @@ export class Buffer {
     return value instanceof Buffer;
   }
 
+  static concat(list: Buffer[], totalLength?: number): Buffer {
+    if (totalLength == undefined) {
+      totalLength = list.reduce((acc, curr) => acc + curr.length, 0)
+    }
+
+    const bytes = Bytes.withCapacity(totalLength)
+    for (const buf of list) {
+      if (bytes.len === bytes.capacity) {
+        break;
+      }
+
+      const remaining = bytes.capacity - bytes.len;
+      if (buf.length <= remaining) {
+        bytes.extend(buf.inner.data);
+      } else {
+        bytes.extend(buf.inner.data.slice(0, remaining));
+      }
+    }
+
+    return new Buffer(bytes)
+  }
+
   #inner: Bytes;
   private constructor(inner: Bytes) {
     this.#inner = inner;
@@ -41,7 +62,11 @@ export class Buffer {
     return this.#inner;
   }
 
+  get length(): number {
+    return this.#inner.len;
+  }
+
   public toString(encoding: Encoding = 'utf8'): string {
     return this.#inner.decode(encoding);
   }
-}
+}

core_js/map-std/src/internal/types.ts

@@ -1,2 +1,2 @@
 export type MultiMap = Record<string, string[]>;
-export type Encoding = 'utf8' | 'base64';
+export type Encoding = 'utf8' | 'base64' | 'base64url';

core_js/map-std/src/map_std.ts

@@ -1,4 +1,4 @@
-import { Buffer as NodeBuffer } from './internal/node_compat';
+import { Buffer as NodeBuffer } from './internal/node_buffer';
 import * as unstable from './unstable';
 
 declare global {

core_js/map-std/src/unstable.ts

@@ -2,7 +2,7 @@ import { messageExchange, jsonReviverMapValue, jsonReplacerMapValue, responseErr
 import { ensureMultimap } from './internal/util';
 import { Bytes, ByteStream } from './internal/bytes';
 import type { MultiMap } from './internal/types';
-import { Buffer } from './internal/node_compat';
+import { Buffer } from './internal/node_buffer';
 
 export type { MultiMap, Encoding } from './internal/types';
 
@@ -53,12 +53,15 @@ export class HttpResponse {
     this.#bodyStream = new ByteStream(bodyStream);
   }
 
-  // TODO: either make Bytes public or use a different type
   private bodyBytes(): Bytes {
-    const buffer = this.#bodyStream.readToEnd();
+    const bytes = this.#bodyStream.readToEnd();
     this.#bodyStream.close();
 
-    return buffer;
+    return bytes;
+  }
+
+  public bodyBuffer(): Buffer {
+    return Buffer.from(this.bodyBytes());
   }
 
   public bodyText(): string {