Enable multiple_members for GzipDecoder (#354)

* Enable `multiple_members` for `GzipDecoder` This commit enables support for `multiple_members` which allows the `GzipDecoder` to support gzip streams consisting of multiple, concatenated gzip's - which in turn is a valid gzip. Before this commit, when using the decompression middleware, the input stream would be closed after reading the first gzip chunk, ignoring the rest of the gzip - meaning the resulting decompressed body could be missing data but still considered "OK". An example of a concatenated gzip: ```bash echo "hello" | gzip > hello.txt.gz echo "world" | gzip > world.txt.gz cat hello.txt.gz world.txt.gz > hello-world.txt.gz gzip -d < hello-world.txt.gz hello world ``` Fixes: #353 * update changelog for #354
tower-rs · Apr 13, 2023 · 92d1954 · 92d1954
1 parent 4788a44
commit 92d1954
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 2 deletions.
diff --git a/tower-http/CHANGELOG.md b/tower-http/CHANGELOG.md
@@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Changed
 
-- None.
+- **decompression:** Enable `multiple_members` for `GzipDecoder` ([#354])
 
 ## Removed
 
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 [#335]: https://github.com/tower-rs/tower-http/pull/335
 [#336]: https://github.com/tower-rs/tower-http/pull/336
+[#354]: https://github.com/tower-rs/tower-http/pull/354
 
 # 0.4.0 (February 24, 2023)
 

diff --git a/tower-http/src/decompression/body.rs b/tower-http/src/decompression/body.rs
@@ -346,7 +346,9 @@ where
     type Output = GzipDecoder<Self::Input>;
 
     fn apply(input: Self::Input, _quality: CompressionLevel) -> Self::Output {
-        GzipDecoder::new(input)
+        let mut decoder = GzipDecoder::new(input);
+        decoder.multiple_members(true);
+        decoder
     }
 
     fn get_pin_mut(pinned: Pin<&mut Self::Output>) -> Pin<&mut Self::Input> {

diff --git a/tower-http/src/decompression/mod.rs b/tower-http/src/decompression/mod.rs
@@ -115,9 +115,12 @@ pub use self::request::service::RequestDecompression;
 
 #[cfg(test)]
 mod tests {
+    use std::io::Write;
+
     use super::*;
     use crate::compression::Compression;
     use bytes::BytesMut;
+    use flate2::write::GzEncoder;
     use http::Response;
     use http_body::Body as _;
     use hyper::{Body, Client, Error, Request};
@@ -145,10 +148,48 @@ mod tests {
         assert_eq!(decompressed_data, "Hello, World!");
     }
 
+    #[tokio::test]
+    async fn decompress_multi_gz() {
+        let mut client = Decompression::new(service_fn(handle_multi_gz));
+
+        let req = Request::builder()
+            .header("accept-encoding", "gzip")
+            .body(Body::empty())
+            .unwrap();
+        let res = client.ready().await.unwrap().call(req).await.unwrap();
+
+        // read the body, it will be decompressed automatically
+        let mut body = res.into_body();
+        let mut data = BytesMut::new();
+        while let Some(chunk) = body.data().await {
+            let chunk = chunk.unwrap();
+            data.extend_from_slice(&chunk[..]);
+        }
+        let decompressed_data = String::from_utf8(data.freeze().to_vec()).unwrap();
+
+        assert_eq!(decompressed_data, "Hello, World!");
+    }
+
     async fn handle(_req: Request<Body>) -> Result<Response<Body>, Error> {
         Ok(Response::new(Body::from("Hello, World!")))
     }
 
+    async fn handle_multi_gz(_req: Request<Body>) -> Result<Response<Body>, Error> {
+        let mut buf = Vec::new();
+        let mut enc1 = GzEncoder::new(&mut buf, Default::default());
+        enc1.write_all(b"Hello, ").unwrap();
+        enc1.finish().unwrap();
+
+        let mut enc2 = GzEncoder::new(&mut buf, Default::default());
+        enc2.write_all(b"World!").unwrap();
+        enc2.finish().unwrap();
+
+        let mut res = Response::new(Body::from(buf));
+        res.headers_mut()
+            .insert("content-encoding", "gzip".parse().unwrap());
+        Ok(res)
+    }
+
     #[allow(dead_code)]
     async fn is_compatible_with_hyper() {
         let mut client = Decompression::new(Client::new());