Correct input_length since habana extend input_length to max_input_le…

…ngth (#103) Signed-off-by: Wang, Yi A <[email protected]>
huggingface · Mar 18, 2024 · d752317 · d752317
1 parent b45f648
commit d752317
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/router/src/validation.rs b/router/src/validation.rs
@@ -97,6 +97,12 @@ impl Validation {
             // Unwrap is safe here
             let (inputs, input_length) = response_receiver.await.unwrap()?;
 
+            let input_length = if self.skip_tokenizer_in_tgi {
+                inputs.chars().filter(|&c| c == ',').count() + 1
+            } else {
+                truncate.unwrap_or(self.max_input_length)
+            };
+
             // Get total tokens
             let max_new_tokens: u32 = if let Some(max_new_tokens) = max_new_tokens {
                 max_new_tokens