update

paddlenlp/transformers/tokenizer_utils_base.py

@@ -1557,8 +1557,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
                             )
         tokenizer_config_file_dir_list = set()
         for k, v in resolved_vocab_files.items():
-            tokenizer_config_file_dir_list.add(os.path.dirname(v))
-        assert len(tokenizer_config_file_dir_list) <= 1, "All tokenizer files should be in the same directory."
+            if v is not None and os.path.isfile(v):
+                tokenizer_config_file_dir_list.add(os.path.dirname(v))
+        assert len(tokenizer_config_file_dir_list) == 1, "All tokenizer files should be in the same directory."
         # Prepare tokenizer initialization kwargs
         # Did we saved some inputs and kwargs to reload ?
         has_tokenizer_file = resolved_vocab_files.get("tokenizer_file", None) is not None