Skip to content

Commit c749cb0

Browse files
eduardopessineduardopessin
authored
llama : make tensor-split regex patterns static (ggml-org#24710)
llama_meta_device_get_split_state() recompiled 29 std::regex on every call. In -sm tensor mode the callback runs once per tensor per token, so this dominated the decode thread in profiling. Mark them static const so they are compiled once. Kept inside the function (local statics are thread-safe since C++11). Patterns are literal and stateless, so behavior is unchanged.
1 parent 67776ea commit c749cb0

1 file changed

Lines changed: 32 additions & 32 deletions

File tree

‎src/llama-model.cpp‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -336,38 +336,38 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
336336
const llama_hparams & hparams = ud->model->hparams;
337337
const std::string tensor_name = tensor->name;
338338

339-
const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
340-
const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
341-
const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
342-
const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
343-
const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
344-
const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
345-
const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
346-
const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
347-
const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
348-
const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
349-
const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
350-
const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
351-
352-
const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
353-
const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
354-
const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
355-
const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
356-
const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
357-
const std::regex pattern_r_cache ("cache_r_l\\d*");
358-
const std::regex pattern_s_cache ("cache_s_l\\d*");
359-
const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
360-
const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
361-
362-
const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
363-
const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
364-
const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
365-
const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
366-
const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
367-
const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");
368-
369-
const std::regex pattern_output_weight("output\\.weight");
370-
const std::regex pattern_output_bias ("output\\.bias");
339+
static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
340+
static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");
341+
static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");
342+
static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");
343+
static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");
344+
static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");
345+
static const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");
346+
static const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");
347+
static const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");
348+
static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");
349+
static const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");
350+
static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");
351+
352+
static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");
353+
static const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");
354+
static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");
355+
static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");
356+
static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");
357+
static const std::regex pattern_r_cache ("cache_r_l\\d*");
358+
static const std::regex pattern_s_cache ("cache_s_l\\d*");
359+
static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");
360+
static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");
361+
362+
static const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");
363+
static const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");
364+
static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");
365+
static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");
366+
static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");
367+
static const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");
368+
369+
static const std::regex pattern_output_weight("output\\.weight");
370+
static const std::regex pattern_output_bias ("output\\.bias");
371371

372372
struct tensor_config {
373373
ggml_backend_meta_split_axis axis;

0 commit comments

Comments
 (0)