Skip to content

Commit

Permalink
better comments
Browse files Browse the repository at this point in the history
  • Loading branch information
gante committed Dec 14, 2023
1 parent 7e4deab commit e234e1e
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions src/transformers/generation/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4629,8 +4629,8 @@ def assisted_decoding(
# 👉 Apply algorithm 1 from the speculative decoding paper (https://arxiv.org/pdf/2211.17192.pdf).
# NOTE: Unless otherwise stated, the variable names match those in the paper.
if do_sample and candidate_logits is not None:
# Gets the probabilities from the logits. q_i and p_i denote the model and assistant (respectively)
# probabilities of the tokens selected by the assistant.
# Gets the probabilities from the logits. q_i and p_i denote the model and assistant probabilities of
# the tokens selected by the assistant, respectivelly.
q = candidate_logits.softmax(dim=-1)
q_i = q[
:,
Expand All @@ -4646,7 +4646,7 @@ def assisted_decoding(
probability_ratio = p_i / q_i

# When probability_ratio > 1 (i.e. q_i(x) < p_i(x)), keep the token. Otherwise reject with
# p = 1 - probability_ratio (= keep with p=probability_ratio). Keep all the tokens until the first
# p = 1 - probability_ratio (= keep with p = probability_ratio). Keep all the tokens until the first
# rejection
r_i = torch.rand_like(probability_ratio)
is_rejected = r_i > probability_ratio # equivalent: is_accepted = r_i <= probability_ratio
Expand Down

0 comments on commit e234e1e

Please sign in to comment.