Skip to content

Commit a4e6821

Browse files
committed
fix: disable GPU forcing/hijack on every token (1.48 t/s bottleneck)
The forcing + hijack + D2D copy + reallocation loop runs on every single-token decode, adding ~600ms/token overhead. Kept as empty if(do_cuda) block for future activation. Baseline hybrid path restored. GPU infrastructure preserved and ready: force + hijack + D2D copy + get_tensor_async fallback all remain. Just needs graph reuse or CUDA graph capture to amortize the per-token cost.
1 parent 2da0fb7 commit a4e6821

1 file changed

Lines changed: 3 additions & 45 deletions

File tree

src/llama-context.cpp

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2106,52 +2106,10 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
21062106
if (res->t_embd) ggml_backend_sched_set_tensor_backend(sched.get(), res->t_embd, be);
21072107

21082108
if (do_cuda) {
2109-
// --- CAPTURE PATH ---
2110-
// 1. Reset + force matched MoE tensors to GPU (cascade to consumers)
2111-
ggml_backend_sched_reset(sched.get());
2112-
for (int i = 0; i < ggml_graph_n_nodes(phase2_gf); i++) {
2113-
ggml_tensor * t = ggml_graph_node(phase2_gf, i);
2114-
auto [tid, il] = h2_hijack.match_name(t->name);
2115-
if (tid >= 0) ggml_backend_sched_set_tensor_backend(sched.get(), t, gpu);
2116-
}
2117-
for (int i = 0; i < ggml_graph_n_leafs(phase2_gf); i++) {
2118-
ggml_tensor * t = ggml_graph_leaf(phase2_gf, i);
2119-
auto [tid, il] = h2_hijack.match_name(t->name);
2120-
if (tid >= 0) ggml_backend_sched_set_tensor_backend(sched.get(), t, gpu);
2121-
}
2122-
// Cascade: force VIEW (op=37) and ADD (op=2) nodes that consume
2123-
// matched tensors (catches expert output combination chain only)
2124-
for (int i = 0; i < ggml_graph_n_nodes(phase2_gf); i++) {
2125-
ggml_tensor * t = ggml_graph_node(phase2_gf, i);
2126-
if (t->op != 37 && t->op != 2) continue;
2127-
for (int s = 0; s < GGML_MAX_SRC && t->src[s]; s++) {
2128-
auto [sid, sil] = h2_hijack.match_name(t->src[s]->name);
2129-
if (sid >= 0) {
2130-
ggml_backend_sched_set_tensor_backend(sched.get(), t, gpu);
2131-
break;
2132-
}
2133-
}
2134-
}
2135-
force_idxs_to_cpu();
2136-
if (!ggml_backend_sched_alloc_graph(sched.get(), phase2_gf)) {
2137-
ret = GGML_STATUS_ALLOC_FAILED; return nullptr;
2138-
}
2139-
if (res->t_logits) ggml_backend_sched_set_tensor_backend(sched.get(), res->t_logits, be);
2140-
if (res->t_embd) ggml_backend_sched_set_tensor_backend(sched.get(), res->t_embd, be);
2141-
2142-
h2_hijack.scan_and_hijack(phase2_gf);
2143-
res->set_inputs(&ubatch);
2144-
void* st = ggml_backend_cuda_get_stream_ptr(gpu, 0);
2145-
h2_hijack.copy_data_to_static(st);
2146-
ggml_backend_sched_synchronize(sched.get());
2147-
auto s = graph_compute(phase2_gf, ubatch.n_tokens > 1);
2148-
if (s != GGML_STATUS_SUCCESS) { ret = s; return nullptr; }
2149-
} else {
2150-
res->set_inputs(&ubatch);
2151-
ggml_backend_sched_synchronize(sched.get());
2152-
auto s = graph_compute(phase2_gf, ubatch.n_tokens > 1);
2153-
if (s != GGML_STATUS_SUCCESS) { ret = s; return nullptr; }
21542109
}
2110+
res->set_inputs(&ubatch);
2111+
ggml_backend_sched_synchronize(sched.get());
2112+
{ auto s = graph_compute(phase2_gf, ubatch.n_tokens > 1); if (s != GGML_STATUS_SUCCESS) { ret = s; return nullptr; } }
21552113
}
21562114
#else
21572115
res->reset();

0 commit comments

Comments
 (0)