small fix to remove unused eos in processor when it's not used. (#23408)

huggingface · May 23, 2023 · e30ceae · e30ceae
1 parent 2f424d7
commit e30ceae
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
@@ -1175,7 +1175,11 @@ def post_processor(self):
             single = f"{(bos+':0 ') * add_bos}$A:0{(' '+eos+':0') * add_eos}"
             pair = f"{single}{(' '+bos+':1') * add_bos} $B:1{(' '+eos+':1') * add_eos}"
 
-            special_tokens = [(bos, bos_token_id), (eos, eos_token_id)]
+            special_tokens = []
+            if add_bos:
+                special_tokens.append((bos, bos_token_id))
+            if add_eos:
+                special_tokens.append((eos, eos_token_id))
             return processors.TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens)
 
         else: