add alpha scaling to lora

examples/nlp/language_modeling/tuning/conf/megatron_gpt_finetuning_config.yaml

@@ -95,6 +95,7 @@ model:
 
     lora_tuning:
       adapter_dim: 32
+      alpha: ${model.peft.lora_tuning.adapter_dim} 
       adapter_dropout: 0.0
       column_init_method: 'xavier' # IGNORED if linear_adapter is used, options: xavier, zero or normal
       row_init_method: 'zero' # IGNORED if linear_adapter is used, options: xavier, zero or normal

nemo/collections/nlp/modules/common/megatron/adapters/parallel_adapters.py

@@ -130,6 +130,7 @@ def __init__(
         gather_output: bool = True,
         dropout: float = 0.0,
         model_parallel_config: Optional[ModelParallelConfig] = None,
+        alpha: float | None = None,
         **kwargs,
     ):
         super().__init__()
@@ -142,6 +143,7 @@ def __init__(
         self.activation = activation_registry[activation]()
         self.norm_position = norm_position
         self.dim = dim
+        self.alpha = alpha if alpha is not None else self.dim
 
         # megatron_gpt_peft_models will provide this arg, but deprecated ones do not.
         # in case this arg is not provided, use the dummy default config.
@@ -235,6 +237,8 @@ def forward(self, x):
         if self.dropout is not None:
             x = self.dropout(x)
 
+        x = x * (self.alpha / self.dim)
+
         return x
 
 
@@ -250,6 +254,7 @@ class ParallelLinearAdapterConfig(AdapterConfig):
     row_init_method: str = 'zero'
     gather_output: bool = True
     dropout: float = 0.0
+    alpha: float | None = None
     network_alpha: int | None = None
     _target_: str = "{0}.{1}".format(ParallelLinearAdapter.__module__, ParallelLinearAdapter.__name__)
 

nemo/collections/nlp/parts/peft_config.py

@@ -74,6 +74,7 @@ def __init__(self, cfg):
         if num_query_groups is None:
             num_query_groups = cfg.num_attention_heads
         qkv_projection_size = projection_size + (2 * kv_channels * num_query_groups)
+        alpha = lora_cfg.get("alpha", lora_cfg.adapter_dim)
 
         config_args = {
             "in_features": cfg.hidden_size,
@@ -86,6 +87,7 @@ def __init__(self, cfg):
             "row_init_method": lora_cfg.get("row_init_method", "zero"),
             "gather_output": False,
             "dropout": lora_cfg.adapter_dropout,
+            "alpha": alpha,
         }
 
         if lora_cfg.weight_tying: