|
18 | 18 |
|
19 | 19 | import torch |
20 | 20 | import torch.nn as nn |
21 | | -import torch.nn.functional as F |
22 | 21 | from torch.nn import LayerNorm |
23 | | - |
24 | 22 | from transformers.activations import ACT2FN |
25 | 23 |
|
26 | 24 |
|
@@ -80,15 +78,11 @@ def __init__( |
80 | 78 | self.embed_dim = embed_dim |
81 | 79 |
|
82 | 80 | kernel_size = [patch_size, patch_size] |
83 | | - self.proj = nn.Conv2d( |
84 | | - in_channels, embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=False |
85 | | - ) |
| 81 | + self.proj = nn.Conv2d(in_channels, embed_dim, kernel_size=kernel_size, stride=kernel_size, bias=False) |
86 | 82 |
|
87 | 83 | def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: |
88 | 84 | target_dtype = self.proj.weight.dtype |
89 | | - hidden_states = hidden_states.view( |
90 | | - -1, self.in_channels, self.patch_size, self.patch_size |
91 | | - ) |
| 85 | + hidden_states = hidden_states.view(-1, self.in_channels, self.patch_size, self.patch_size) |
92 | 86 | hidden_states = self.proj(hidden_states.to(dtype=target_dtype)).view(-1, self.embed_dim) |
93 | 87 | return hidden_states |
94 | 88 |
|
@@ -148,12 +142,7 @@ def forward( |
148 | 142 | position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, |
149 | 143 | ) -> torch.Tensor: |
150 | 144 | seq_length = hidden_states.shape[0] |
151 | | - q, k, v = ( |
152 | | - self.qkv(hidden_states) |
153 | | - .reshape(seq_length, 3, self.num_heads, -1) |
154 | | - .permute(1, 0, 2, 3) |
155 | | - .unbind(0) |
156 | | - ) |
| 145 | + q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) |
157 | 146 |
|
158 | 147 | if position_embeddings is None: |
159 | 148 | emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) |
|
0 commit comments