diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
index 26934f1cc8f89..2531580d45650 100755
--- a/convert-hf-to-gguf.py
+++ b/convert-hf-to-gguf.py
@@ -929,11 +929,11 @@ def write_tensors(self):
 
 class StableLMModel(Model):
     def set_vocab(self):
-        # StableLM 2 1.6B uses a vocab in a similar format to Qwen's vocab
-        if self.hparams["vocab_size"] > 100000:
-            self._set_vocab_qwen()
-        else:
+        if (self.dir_model / "tokenizer.json").is_file():
             self._set_vocab_gpt2()
+        else:
+            # StableLM 2 1.6B uses a vocab in a similar format to Qwen's vocab
+            self._set_vocab_qwen()
 
     def set_gguf_parameters(self):
         hparams = self.hparams