Enable special token support for exllamav2

modules/exllamav2.py

@@ -64,15 +64,15 @@ def from_pretrained(self, path_to_model):
         return result, result
 
     def encode(self, string, **kwargs):
-        return self.tokenizer.encode(string, add_bos=True)
+        return self.tokenizer.encode(string, add_bos=True, encode_special_tokens=True)
 
     def decode(self, ids, **kwargs):
         if isinstance(ids, list):
             ids = torch.tensor([ids])
         elif isinstance(ids, torch.Tensor) and ids.numel() == 1:
             ids = ids.view(1, -1)
 
-        return self.tokenizer.decode(ids)[0]
+        return self.tokenizer.decode(ids, decode_special_tokens=True)[0]
 
     def get_logits(self, token_ids, **kwargs):
         self.cache.current_seq_len = 0
@@ -97,7 +97,7 @@ def generate_with_streaming(self, prompt, state):
             if len(to_ban) > 0:
                 settings.disallow_tokens(self.tokenizer, to_ban)
 
-        ids = self.tokenizer.encode(prompt, add_bos=state['add_bos_token'])
+        ids = self.tokenizer.encode(prompt, add_bos=state['add_bos_token'], encode_special_tokens=True)
         ids = ids[:, -get_max_prompt_length(state):]
         initial_len = ids.shape[-1]
 
@@ -119,7 +119,7 @@ def generate_with_streaming(self, prompt, state):
             if i == 0 and self.tokenizer.tokenizer.IdToPiece(int(token)).startswith('▁'):
                 has_leading_space = True
 
-            decoded_text = self.tokenizer.decode(ids[:, initial_len:])[0]
+            decoded_text = self.tokenizer.decode(ids[:, initial_len:], decode_special_tokens=not state['skip_special_tokens'])[0]
             if has_leading_space:
                 decoded_text = ' ' + decoded_text
 

modules/loaders.py

@@ -231,6 +231,7 @@
         'ban_eos_token',
         'add_bos_token',
         'custom_token_bans',
+        'skip_special_tokens',
         'auto_max_new_tokens',
     },
     'ExLlamav2_HF': {