fix: remove extra prepare model when enable tokenization flag in some…

… ops (#99)
modelscope · Nov 24, 2023 · 754720a · 754720a
1 parent 155f9f6
commit 754720a
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/data_juicer/utils/model_utils.py b/data_juicer/utils/model_utils.py
@@ -269,6 +269,4 @@ def get_model(model_key, lang='en', model_type='sentencepiece'):
 
     :param model_key: name of the model or tokenzier
     """
-    if model_key not in MODEL_ZOO:
-        prepare_model(lang=lang, model_type=model_type, model_key=model_key)
     return MODEL_ZOO.get(model_key, None)