feat(node-http-handler): improve stream collection performance (#1272)

* feat(node-http-handler): improve stream collection performance * changeset * feat: update stream-collector implementation
smithy-lang · May 10, 2024 · e76e736 · e76e736
1 parent 5060cde
commit e76e736
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 12 deletions.
diff --git a/.changeset/fluffy-crabs-watch.md b/.changeset/fluffy-crabs-watch.md
@@ -0,0 +1,6 @@
+---
+"@smithy/fetch-http-handler": patch
+"@smithy/node-http-handler": patch
+---
+
+improve stream collection speed
diff --git a/packages/fetch-http-handler/src/stream-collector.ts b/packages/fetch-http-handler/src/stream-collector.ts
@@ -17,20 +17,28 @@ async function collectBlob(blob: Blob): Promise<Uint8Array> {
 }
 
 async function collectStream(stream: ReadableStream): Promise<Uint8Array> {
-  let res = new Uint8Array(0);
+  const chunks = [];
   const reader = stream.getReader();
   let isDone = false;
+  let length = 0;
+
   while (!isDone) {
     const { done, value } = await reader.read();
     if (value) {
-      const prior = res;
-      res = new Uint8Array(prior.length + value.length);
-      res.set(prior);
-      res.set(value, prior.length);
+      chunks.push(value);
+      length += value.length;
     }
     isDone = done;
   }
-  return res;
+
+  const collected = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    collected.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return collected;
 }
 
 function readToBase64(blob: Blob): Promise<string> {

diff --git a/packages/node-http-handler/src/stream-collector/index.ts b/packages/node-http-handler/src/stream-collector/index.ts
@@ -35,18 +35,26 @@ const isReadableStreamInstance = (stream: unknown): stream is IReadableStream =>
   typeof ReadableStream === "function" && stream instanceof ReadableStream;
 
 async function collectReadableStream(stream: IReadableStream): Promise<Uint8Array> {
-  let res = new Uint8Array(0);
+  const chunks = [];
   const reader = stream.getReader();
   let isDone = false;
+  let length = 0;
+
   while (!isDone) {
     const { done, value } = await reader.read();
     if (value) {
-      const prior = res;
-      res = new Uint8Array(prior.length + value.length);
-      res.set(prior);
-      res.set(value, prior.length);
+      chunks.push(value);
+      length += value.length;
     }
     isDone = done;
   }
-  return res;
+
+  const collected = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    collected.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return collected;
 }