Skip to content

Commit 46394ed

Browse files
committed
fix: force ALL kq_mask variants to CPU in force_idxs_to_cpu (mask tensors require host buffer)
1 parent dbbcf51 commit 46394ed

1 file changed

Lines changed: 7 additions & 1 deletion

File tree

src/llama-context.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1699,20 +1699,26 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
16991699
auto * base = inp.get();
17001700
if (auto * akv = dynamic_cast<llm_graph_input_attn_kv *>(base)) {
17011701
force(akv->self_k_idxs); force(akv->self_v_idxs);
1702+
force(akv->self_kq_mask); force(akv->self_kq_mask_swa);
17021703
} else if (auto * ak = dynamic_cast<llm_graph_input_attn_k *>(base)) {
1703-
force(ak->self_k_idxs);
1704+
force(ak->self_k_idxs); force(ak->self_kq_mask);
17041705
} else if (auto * dsa = dynamic_cast<llm_graph_input_attn_k_dsa *>(base)) {
17051706
force(dsa->self_k_idxs_mla); force(dsa->self_k_idxs_lid);
1707+
force(dsa->self_kq_mask_mla); force(dsa->self_kq_mask_lid);
17061708
} else if (auto * iswa = dynamic_cast<llm_graph_input_attn_kv_iswa *>(base)) {
17071709
force(iswa->self_k_idxs); force(iswa->self_v_idxs);
17081710
force(iswa->self_k_idxs_swa); force(iswa->self_v_idxs_swa);
1711+
force(iswa->self_kq_mask); force(iswa->self_kq_mask_swa);
17091712
} else if (auto * hyb = dynamic_cast<llm_graph_input_mem_hybrid *>(base)) {
17101713
force(hyb->inp_attn->self_k_idxs); force(hyb->inp_attn->self_v_idxs);
1714+
force(hyb->inp_attn->self_kq_mask);
17111715
} else if (auto * hybk = dynamic_cast<llm_graph_input_mem_hybrid_k *>(base)) {
17121716
force(hybk->inp_attn->self_k_idxs);
1717+
force(hybk->inp_attn->self_kq_mask);
17131718
} else if (auto * hybiswa = dynamic_cast<llm_graph_input_mem_hybrid_iswa *>(base)) {
17141719
force(hybiswa->inp_attn->self_k_idxs); force(hybiswa->inp_attn->self_v_idxs);
17151720
force(hybiswa->inp_attn->self_k_idxs_swa); force(hybiswa->inp_attn->self_v_idxs_swa);
1721+
force(hybiswa->inp_attn->self_kq_mask); force(hybiswa->inp_attn->self_kq_mask_swa);
17161722
} else if (auto * oid = dynamic_cast<llm_graph_input_out_ids *>(base)) {
17171723
force(oid->out_ids);
17181724
}

0 commit comments

Comments
 (0)