Skip to content

Commit 7a988c3

Browse files
Fix trailing whitespace in attention matrix computation
Co-authored-by: meilame-tayebjee <114609737+meilame-tayebjee@users.noreply.github.com>
1 parent 87d672f commit 7a988c3

1 file changed

Lines changed: 2 additions & 2 deletions

File tree

torchTextClassifiers/model/components/text_embedder.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -383,14 +383,14 @@ def forward(self, token_embeddings, attention_mask: Optional[torch.Tensor] = Non
383383
# Compute attention scores
384384
# size (B, n_head, n_labels, seq_len)
385385
attention_scores = torch.matmul(q, k.transpose(-2, -1)) / (self.head_dim**0.5)
386-
386+
387387
# Apply mask to attention scores before softmax
388388
if attention_mask is not None:
389389
# attn_mask is already in the right shape: (B, 1, 1, T)
390390
# We need to apply it to scores of shape (B, n_head, n_labels, T)
391391
# Set masked positions to -inf so they become 0 after softmax
392392
attention_scores = attention_scores.masked_fill(attn_mask, float('-inf'))
393-
393+
394394
attention_matrix = torch.softmax(attention_scores, dim=-1)
395395

396396
return {"sentence_embedding": y, "attention_matrix": attention_matrix}

0 commit comments

Comments
 (0)