add torch_native_sink backend and enable decode with sink (mingfeima#57)

Author: blzheng

python/sglang/srt/layers/activation.py

@@ -135,6 +135,19 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = F.relu(x)
         return x * x
 
+class SwiGLU(CustomOp):
+    def forward_native(self, x: torch.Tensor, alpha: float = 1.702, pair_wise: bool = True) -> torch.Tensor:
+        # reference implementation
+        if not pair_wise:
+            x_glu, x_linear = torch.chunk(x, 2, dim=-1)
+        else:
+            x_glu, x_linear = x[..., ::2], x[..., 1::2]
+        out_glu = x_glu * torch.sigmoid(alpha * x_glu)
+        return out_glu * (x_linear + 1) # Note that here add an extra bias of 1 to the linear layer
+
+    def forward_cuda(self, x: torch.Tensor, alpha: float = 1.702, pair_wise: bool = True) -> torch.Tensor:
+        # TODO: Implement the CUDA kernel for SwiGLU in sgl-kernel
+        return self.forward_native(x, alpha, pair_wise)
 
 class QuickGELU(CustomOp):
     def forward_native(self, x: torch.Tensor) -> torch.Tensor:

python/sglang/srt/layers/attention/intel_amx_backend.py

@@ -3,7 +3,6 @@
 from typing import TYPE_CHECKING
 
 import torch
-from torch.nn.functional import scaled_dot_product_attention
 
 from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
 from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -50,187 +49,6 @@ def init_forward_metadata(self, forward_batch: ForwardBatch):
             max_extend_len = torch.max(forward_batch.extend_seq_lens).item()
         self.forward_metadata = (attn_logits, max_extend_len)
 
-    def get_graph_seq_len_fill_value(self):
-        return 1
-
-    
-    def _run_sdpa_forward_extend(
-        self,
-        query: torch.Tensor,
-        output: torch.Tensor,
-        k_cache: torch.Tensor,
-        v_cache: torch.Tensor,
-        req_to_token: torch.Tensor,
-        req_pool_indices: torch.Tensor,
-        seq_lens: torch.Tensor,
-        extend_prefix_lens: torch.Tensor,
-        extend_seq_lens: torch.Tensor,
-        encoder_lens=None,
-        scaling=None,
-        enable_gqa=False,
-        causal=False,
-        is_cross_attn=False,
-    ):
-        """Run the extend forward by using torch native sdpa op.
-
-        Args:
-            query: [num_tokens, num_heads, head_size]
-            output: [num_tokens, num_heads, head_size]
-            k_cache: [max_total_num_tokens, num_heads, head_size]
-            v_cache: [max_total_num_tokens, num_heads, head_size]
-            req_to_token: [max_num_reqs, max_context_len]
-            req_pool_indices: [num_seqs]
-            encoder_lens: [num_seqs] or None
-            seq_lens: [num_seqs]
-            extend_prefix_lens: [num_seqs]
-            extend_seq_lens: [num_seqs]
-            scaling: float or None
-            enable_gqa: bool
-            causal: bool
-            is_cross_attn: bool
-
-        Returns:
-            output: [num_tokens, num_heads, head_size]
-        """
-
-        assert seq_lens.shape[0] == extend_prefix_lens.shape[0]
-        assert seq_lens.shape[0] == extend_seq_lens.shape[0]
-
-        # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size]
-        query = query.movedim(0, query.dim() - 2)
-
-        start_q, start_kv = 0, 0
-        for seq_idx in range(seq_lens.shape[0]):
-            # TODO: this loop process a sequence per iter, this is inefficient.
-            # Need optimize the performance later.
-
-            extend_seq_len_q = extend_seq_lens[seq_idx]
-            prefill_seq_len_q = extend_prefix_lens[seq_idx]
-
-            seq_len_kv = seq_lens[seq_idx]
-            end_q = start_q + extend_seq_len_q
-            if encoder_lens is not None:
-                start_kv = 0 if is_cross_attn else encoder_lens[seq_idx]
-                end_kv = (
-                    encoder_lens[seq_idx] if is_cross_attn else start_kv + seq_len_kv
-                )
-            else:
-                start_kv = 0
-                end_kv = start_kv + seq_len_kv
-            per_req_query = query[:, start_q:end_q, :]
-            per_req_query_redudant = torch.empty(
-                (per_req_query.shape[0], seq_len_kv, per_req_query.shape[2]),
-                dtype=per_req_query.dtype,
-                device=per_req_query.device,
-            )
-
-            per_req_query_redudant[:, prefill_seq_len_q:, :] = per_req_query
-
-            # get key and value from cache. per_req_tokens contains the kv cache
-            # index for each token in the sequence.
-            req_pool_idx = req_pool_indices[seq_idx]
-            per_req_tokens = req_to_token[req_pool_idx, start_kv:end_kv]
-            per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2)
-            per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2)
-
-            per_req_out_redudant = (
-                scaled_dot_product_attention(
-                    per_req_query_redudant.unsqueeze(0),
-                    per_req_key.unsqueeze(0),
-                    per_req_value.unsqueeze(0),
-                    enable_gqa=enable_gqa,
-                    scale=scaling,
-                    is_causal=causal,
-                )
-                .squeeze(0)
-                .movedim(query.dim() - 2, 0)
-            )
-            output[start_q:end_q, :, :] = per_req_out_redudant[prefill_seq_len_q:, :, :]
-            start_q, start_kv = end_q, end_kv
-        return output
-
-    def _run_sdpa_forward_decode(
-        self,
-        query: torch.Tensor,
-        output: torch.Tensor,
-        k_cache: torch.Tensor,
-        v_cache: torch.Tensor,
-        req_to_token: torch.Tensor,
-        req_pool_indices: torch.Tensor,
-        seq_lens: torch.Tensor,
-        encoder_lens=None,
-        scaling=None,
-        enable_gqa=False,
-        causal=False,
-        is_cross_attn=False,
-    ):
-        """Run the decode forward by using torch native sdpa op.
-
-        Args:
-            query: [num_tokens, num_heads, head_size]
-            output: [num_tokens, num_heads, head_size]
-            k_cache: [max_total_num_tokens, num_heads, head_size]
-            v_cache: [max_total_num_tokens, num_heads, head_size]
-            req_to_token: [max_num_reqs, max_context_len],
-            req_pool_indices: [num_seqs],
-            seq_lens: [num_seqs]
-            encoder_lens: [num_seqs] or None
-            scaling: float or None
-            enable_gqa: bool
-            causal: bool
-            is_cross_attn: bool
-
-        Returns:
-            output: [num_tokens, num_heads, head_size]
-        """
-
-        # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size]
-        query = query.movedim(0, query.dim() - 2)
-
-        start_q, start_kv = 0, 0
-        for seq_idx in range(seq_lens.shape[0]):
-            # TODO: this loop process a sequence per iter, this is inefficient.
-            # Need optimize the performance later.
-
-            seq_len_q = 1
-            seq_len_kv = seq_lens[seq_idx]
-            end_q = start_q + seq_len_q
-            if encoder_lens is not None:
-                start_kv = 0 if is_cross_attn else encoder_lens[seq_idx]
-                end_kv = (
-                    encoder_lens[seq_idx] if is_cross_attn else start_kv + seq_len_kv
-                )
-            else:
-                start_kv = 0
-                end_kv = start_kv + seq_len_kv
-
-            per_req_query = query[:, start_q:end_q, :]
-
-            # get key and value from cache. per_req_tokens contains the kv cache
-            # index for each token in the sequence.
-
-            req_pool_idx = req_pool_indices[seq_idx]
-            per_req_tokens = req_to_token[req_pool_idx, start_kv:end_kv]
-            per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2)
-            per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2)
-
-            per_req_out = (
-                scaled_dot_product_attention(
-                    per_req_query.unsqueeze(0),
-                    per_req_key.unsqueeze(0),
-                    per_req_value.unsqueeze(0),
-                    enable_gqa=enable_gqa,
-                    scale=scaling,
-                    is_causal=causal,
-                )
-                .squeeze(0)
-                .movedim(query.dim() - 2, 0)
-            )
-            output[start_q:end_q, :, :] = per_req_out
-            start_q, start_kv = end_q, end_kv
-
-        return output
-
     def forward_extend(
         self,
         q,
@@ -239,6 +57,7 @@ def forward_extend(
         layer: RadixAttention,
         forward_batch: ForwardBatch,
         save_kv_cache=True,
+        sk=None,
     ):
         if layer.qk_head_dim != layer.v_head_dim:
             o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim))
@@ -255,46 +74,24 @@ def forward_extend(
                 forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v)
 
         _, max_extend_len = self.forward_metadata
-        if k is not None:
-            assert v is not None
-            self.extend_attention_fwd(
-                q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
-                k,
-                v,
-                o.view(-1, layer.tp_q_head_num, layer.v_head_dim),
-                forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
-                forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
-                forward_batch.req_to_token_pool.req_to_token,
-                forward_batch.req_pool_indices,
-                forward_batch.seq_lens,
-                forward_batch.extend_seq_lens,
-                forward_batch.extend_start_loc,
-                max_extend_len,
-                layer.scaling,
-                layer.logit_cap,
-                forward_batch.encoder_lens,
-            )
-        else:
-            use_gqa = layer.tp_q_head_num != layer.tp_k_head_num
-            q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim)
-            o_ = o.view(-1, layer.tp_q_head_num, layer.v_head_dim)
-
-            self._run_sdpa_forward_extend(
-                q_,
-                o_,
-                forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
-                forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
-                forward_batch.req_to_token_pool.req_to_token,
-                forward_batch.req_pool_indices,
-                forward_batch.seq_lens,
-                forward_batch.extend_prefix_lens,
-                forward_batch.extend_seq_lens,
-                encoder_lens=forward_batch.encoder_lens,
-                scaling=layer.scaling,
-                enable_gqa=use_gqa,
-                causal=not layer.is_cross_attention,
-                is_cross_attn=layer.is_cross_attention,
-            )
+        self.extend_attention_fwd(
+            q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
+            k,
+            v,
+            o.view(-1, layer.tp_q_head_num, layer.v_head_dim),
+            forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
+            forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
+            forward_batch.req_to_token_pool.req_to_token,
+            forward_batch.req_pool_indices,
+            forward_batch.seq_lens,
+            forward_batch.extend_seq_lens,
+            forward_batch.extend_start_loc,
+            max_extend_len,
+            layer.scaling,
+            layer.logit_cap,
+            layer.is_cross_attention,
+            forward_batch.encoder_lens,
+        )
         return o
 
     def forward_decode(
@@ -305,6 +102,7 @@ def forward_decode(
         layer: RadixAttention,
         forward_batch: ForwardBatch,
         save_kv_cache=True,
+        sk=None,
     ):
         attn_logits, _ = self.forward_metadata
 
@@ -319,45 +117,23 @@ def forward_decode(
             if not layer.is_cross_attention
             else forward_batch.encoder_out_cache_loc
         )
-        if k is not None:
-            assert v is not None
-            self.decode_attention_fwd(
-                q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
-                forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
-                forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
-                o.view(-1, layer.tp_q_head_num, layer.v_head_dim),
-                k,
-                v,
-                cache_loc,
-                attn_logits,
-                forward_batch.req_to_token_pool.req_to_token,
-                forward_batch.req_pool_indices,
-                forward_batch.seq_lens,
-                layer.scaling,
-                layer.logit_cap,
-                forward_batch.encoder_lens,
-            )
-        else:
-            use_gqa = layer.tp_q_head_num != layer.tp_k_head_num
-
-            q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim)
-            o_ = o.view(-1, layer.tp_q_head_num, layer.v_head_dim)
-
-            self._run_sdpa_forward_decode(
-                q_,
-                o_,
-                forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
-                forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
-                forward_batch.req_to_token_pool.req_to_token,
-                forward_batch.req_pool_indices,
-                forward_batch.seq_lens,
-                encoder_lens=forward_batch.encoder_lens,
-                scaling=layer.scaling,
-                enable_gqa=use_gqa,
-                causal=False,
-                is_cross_attn=layer.is_cross_attention,
-            )
-
+        self.decode_attention_fwd(
+            q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
+            forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
+            forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
+            o.view(-1, layer.tp_q_head_num, layer.v_head_dim),
+            k,
+            v,
+            cache_loc,
+            attn_logits,
+            forward_batch.req_to_token_pool.req_to_token,
+            forward_batch.req_pool_indices,
+            forward_batch.seq_lens,
+            layer.scaling,
+            layer.logit_cap,
+            layer.is_cross_attention,
+            forward_batch.encoder_lens,
+        )
         return o
 
     def support_triton(self):