update

PaddlePaddle · Jun 21, 2024 · e40bdf1 · e40bdf1
1 parent 827f351
commit e40bdf1
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/llm/predictor.py b/llm/predictor.py
@@ -1215,7 +1215,7 @@ def create_predictor(
 
     # TODO(wj-Mcat): fix llama tokenzier pad_token bug
     if (isinstance(tokenizer, LlamaTokenizer) or isinstance(tokenizer, Llama3Tokenizer)) and not tokenizer.pad_token:
-        tokenizer.pad_token = tokenizer.unk_token
+        tokenizer.pad_token = tokenizer.bos_token
 
     config = AutoConfig.from_pretrained(predictor_args.model_name_or_path)
 

diff --git a/paddlenlp/experimental/transformers/llama/modeling.py b/paddlenlp/experimental/transformers/llama/modeling.py
@@ -919,7 +919,7 @@ def get_cache_kvs_shape(
                 [
                     2,
                     max_batch_size,
-                    config.num_attention_heads // max(config.tensor_parallel_degree, 1),
+                    config.num_key_value_heads // max(config.tensor_parallel_degree, 1),
                     max_length,
                     config.hidden_size // config.num_attention_heads,
                 ]