Skip to content

Commit

Permalink
Fix typo in a parameter name for open llama model (#23637)
Browse files Browse the repository at this point in the history
* Update modeling_open_llama.py

Fix typo in `use_memorry_efficient_attention` parameter name

* Update configuration_open_llama.py

Fix typo in `use_memorry_efficient_attention` parameter name

* Update configuration_open_llama.py

Take care of backwards compatibility ensuring that the previous parameter name is taken into account if used

* Update configuration_open_llama.py

format to adjust the line length

* Update configuration_open_llama.py

proper code formatting using `make fixup`

* Update configuration_open_llama.py

pop the argument not to let it be set later down the line
  • Loading branch information
aaalexlit authored May 23, 2023
1 parent 527ab89 commit b687af0
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def __init__(
bos_token_id=1,
eos_token_id=2,
tie_word_embeddings=False,
use_memorry_efficient_attention=True,
use_memory_efficient_attention=True,
hidden_dropout_prob=0.1,
attention_dropout_prob=0.1,
use_stable_embedding=True,
Expand All @@ -116,7 +116,9 @@ def __init__(
self.initializer_range = initializer_range
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.use_memorry_efficient_attention = use_memorry_efficient_attention
self.use_memory_efficient_attention = kwargs.pop(
"use_memorry_efficient_attention", use_memory_efficient_attention
)
self.hidden_dropout_prob = hidden_dropout_prob
self.attention_dropout_prob = attention_dropout_prob
self.use_stable_embedding = use_stable_embedding
Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/open_llama/modeling_open_llama.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
except ImportError:
xops = None
logger.warn(
"Xformers is not installed correctly. If you want to use memorry_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers."
"Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers."
)


Expand Down Expand Up @@ -223,7 +223,7 @@ def forward(

past_key_value = (key_states, value_states) if use_cache else None

if self.config.use_memorry_efficient_attention and xops is not None and self.training:
if self.config.use_memory_efficient_attention and xops is not None and self.training:
attn_weights = None
query_states = query_states.transpose(1, 2)
key_states = key_states.transpose(1, 2)
Expand Down Expand Up @@ -563,7 +563,7 @@ def forward(
if self.embed_layer_norm:
inputs_embeds = self.embed_layer_norm(inputs_embeds)
# embed positions
if self.config.use_memorry_efficient_attention and self.training:
if self.config.use_memory_efficient_attention and self.training:
attention_mask = None
elif attention_mask is None:
attention_mask = torch.ones(
Expand Down

0 comments on commit b687af0

Please sign in to comment.