From eafceb6dc5b8d74ce4dd087e0a534f5b88a8280c Mon Sep 17 00:00:00 2001
From: SylarTiaNII <15840554235@163.com>
Date: Tue, 7 May 2024 11:46:27 +0800
Subject: [PATCH] [LLM] fix lora target modules on llama

---
 llm/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llm/utils.py b/llm/utils.py
index 8bcc52ae33ab..e2da4b74030d 100644
--- a/llm/utils.py
+++ b/llm/utils.py
@@ -125,9 +125,11 @@ def get_lora_target_modules(model):
             ".*v_proj.*",
             ".*k_proj.*",
             ".*o_proj.*",
+            ".*qkv_proj.*",
             ".*gate_proj.*",
             ".*down_proj.*",
             ".*up_proj.*",
+            ".*gate_up_fused_proj.*",
         ]
     elif model.base_model_prefix == "opt":
         target_modules = [