diff --git a/common/types/model.go b/common/types/model.go index 4a58d1ba..28daa652 100644 --- a/common/types/model.go +++ b/common/types/model.go @@ -282,6 +282,11 @@ type ModelRunReq struct { // When true, the system checks the model metadata for PD recommendation, // validates hardware resources, and splits resources between prefill and decode. EnablePD bool `json:"enable_pd"` + // PD is the client-provided PD (Prefill-Decode) disaggregation configuration. + // When EnablePD is true, the client sends TP/DP/EP/PodsSize for prefill and decode + // roles. The server validates the config against available hardware resources + // instead of deriving it from PDRecommendation. + PD *PDConfig `json:"pd,omitempty"` // OwnerNamespace is optional. If set, the inference is created under this namespace (user or org) for billing and listing; path {namespace} remains the model's owner. OwnerNamespace string `json:"owner_namespace,omitempty"` } @@ -312,6 +317,11 @@ type InstanceRunReq struct { // When true, the system checks the model metadata for PD recommendation, // validates hardware resources, and splits resources between prefill and decode. EnablePD bool `json:"enable_pd"` + // PD is the client-provided PD (Prefill-Decode) disaggregation configuration. + // When EnablePD is true, the client sends TP/DP/EP/PodsSize for prefill and decode + // roles. The server validates the config against available hardware resources + // instead of deriving it from PDRecommendation. + PD *PDConfig `json:"pd,omitempty"` // OwnerNamespace is optional. If set, the finetune is created under this namespace (user or org); path {namespace} remains the model's owner. OwnerNamespace string `json:"owner_namespace,omitempty"` } diff --git a/common/types/pd.go b/common/types/pd.go index 687a37c3..fcba5768 100644 --- a/common/types/pd.go +++ b/common/types/pd.go @@ -190,8 +190,8 @@ type PDConfig struct { } // PDRoleRuntimeConfig holds the runtime parallelism and hardware resource -// configuration for one PD role (prefill or decode). This is derived from -// the PDRecommendation and the user's hardware input, split proportionally. +// configuration for one PD role (prefill or decode). This is provided by the +// client and validated/supplemented by the server. type PDRoleRuntimeConfig struct { // TP is the tensor parallelism degree. TP int `json:"tp"` @@ -199,8 +199,12 @@ type PDRoleRuntimeConfig struct { EP int `json:"ep"` // DP is the data parallelism degree. DP int `json:"dp"` - // TotalGPUs is the total GPUs for this role (TP * EP * DP). + // TotalGPUs is the total GPUs for this role (TP * DP). EP does not add extra GPUs. TotalGPUs int `json:"total_gpus"` + // PodsSize is the number of pods per LWS group (maps to LWS spec.leaderWorkerTemplate.size). + // Each pod runs one vLLM/SGLang instance. GPUs per pod = TotalGPUs / PodsSize. + // When PodsSize is 0 or 1, all GPUs are in a single pod. + PodsSize int `json:"pods_size,omitempty"` // Hardware is the hardware resource allocated to this role. Hardware HardWare `json:"hardware"` }