From e30ceae07b9f49752623646181ed38cc53c699d5 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Tue, 23 May 2023 09:27:36 +0200
Subject: [PATCH] small fix to remove unused eos in processor when it's not
 used. (#23408)

---
 src/transformers/convert_slow_tokenizer.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py
index 195d09ecd89185..1934e35a575d14 100644
--- a/src/transformers/convert_slow_tokenizer.py
+++ b/src/transformers/convert_slow_tokenizer.py
@@ -1175,7 +1175,11 @@ def post_processor(self):
             single = f"{(bos+':0 ') * add_bos}$A:0{(' '+eos+':0') * add_eos}"
             pair = f"{single}{(' '+bos+':1') * add_bos} $B:1{(' '+eos+':1') * add_eos}"
 
-            special_tokens = [(bos, bos_token_id), (eos, eos_token_id)]
+            special_tokens = []
+            if add_bos:
+                special_tokens.append((bos, bos_token_id))
+            if add_eos:
+                special_tokens.append((eos, eos_token_id))
             return processors.TemplateProcessing(single=single, pair=pair, special_tokens=special_tokens)
 
         else: