merge main

Author: shihaobai

lightllm/models/neo_chat_moe/layer_infer/transformer_layer_infer.py

@@ -108,12 +108,13 @@ def _get_qkv_mergekv(
     ):
         input = input.view(-1, self.embed_dim_)
 
-        q = layer_weight.q_proj.mm(input)  # [T, Hq*D]
+        qkv = layer_weight.qkv_proj.mm(input)
+        q, cache_kv = qkv.split(
+            [self.tp_q_head_num_ * self.head_dim_, (self.tp_k_head_num_ + self.tp_v_head_num_) * self.head_dim_], dim=-1
+        )
         q_hw = layer_weight.q_hw_proj.mm(input)
         k_hw = layer_weight.k_hw_proj.mm(input)
 
-        cache_kv = layer_weight.kv_proj.mm(input)  # [T, (Hk+Hv)*D]
-
         layer_weight.q_norm_weight_(q, eps=self.eps_)
         layer_weight.q_norm_hw_weight_(q_hw, eps=self.eps_)
         layer_weight.k_norm_hw_weight_(k_hw, eps=self.eps_)

lightllm/server/api_start.py

@@ -91,7 +91,6 @@ def _set_envs_and_config(args: StartArgs):
 def _launch_subprocesses(args: StartArgs):
 
     _set_envs_and_config(args)
-    set_unique_server_name(args)
 
     if not args.disable_shm_warning:
         check_recommended_shm_size(args)
@@ -291,6 +290,8 @@ def _launch_subprocesses(args: StartArgs):
     args.pd_p_allowed_port_min = 20000
     args.pd_p_allowed_port_max = 30000
 
+    set_unique_server_name(args)
+
     # p d 分离模式下，decode节点的调度间隙是0
     if args.run_mode == "decode":
         args.router_max_wait_tokens = 0