set default batch timeout to 1 ms (#229)

Lightning-AI · Aug 27, 2024 · 6c49137 · 6c49137
1 parent 2bfebcb
commit 6c49137
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/litserve/server.py b/src/litserve/server.py
@@ -417,7 +417,7 @@ def __init__(
         workers_per_device: int = 1,
         timeout: Union[float, bool] = 30,
         max_batch_size: int = 1,
-        batch_timeout: float = 0.0,
+        batch_timeout: float = 0.001,
         api_path: str = "/predict",
         stream: bool = False,
         spec: Optional[LitSpec] = None,