Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/types/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,11 @@ type ModelRunReq struct {
// When true, the system checks the model metadata for PD recommendation,
// validates hardware resources, and splits resources between prefill and decode.
EnablePD bool `json:"enable_pd"`
// PD is the client-provided PD (Prefill-Decode) disaggregation configuration.
// When EnablePD is true, the client sends TP/DP/EP/PodsSize for prefill and decode
// roles. The server validates the config against available hardware resources
// instead of deriving it from PDRecommendation.
PD *PDConfig `json:"pd,omitempty"`
// OwnerNamespace is optional. If set, the inference is created under this namespace (user or org) for billing and listing; path {namespace} remains the model's owner.
OwnerNamespace string `json:"owner_namespace,omitempty"`
}
Expand Down Expand Up @@ -312,6 +317,11 @@ type InstanceRunReq struct {
// When true, the system checks the model metadata for PD recommendation,
// validates hardware resources, and splits resources between prefill and decode.
EnablePD bool `json:"enable_pd"`
// PD is the client-provided PD (Prefill-Decode) disaggregation configuration.
// When EnablePD is true, the client sends TP/DP/EP/PodsSize for prefill and decode
// roles. The server validates the config against available hardware resources
// instead of deriving it from PDRecommendation.
PD *PDConfig `json:"pd,omitempty"`
// OwnerNamespace is optional. If set, the finetune is created under this namespace (user or org); path {namespace} remains the model's owner.
OwnerNamespace string `json:"owner_namespace,omitempty"`
}
Expand Down
10 changes: 7 additions & 3 deletions common/types/pd.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,17 +190,21 @@ type PDConfig struct {
}

// PDRoleRuntimeConfig holds the runtime parallelism and hardware resource
// configuration for one PD role (prefill or decode). This is derived from
// the PDRecommendation and the user's hardware input, split proportionally.
// configuration for one PD role (prefill or decode). This is provided by the
// client and validated/supplemented by the server.
type PDRoleRuntimeConfig struct {
// TP is the tensor parallelism degree.
TP int `json:"tp"`
// EP is the expert parallelism degree (1 for dense models).
EP int `json:"ep"`
// DP is the data parallelism degree.
DP int `json:"dp"`
// TotalGPUs is the total GPUs for this role (TP * EP * DP).
// TotalGPUs is the total GPUs for this role (TP * DP). EP does not add extra GPUs.
TotalGPUs int `json:"total_gpus"`
// PodsSize is the number of pods per LWS group (maps to LWS spec.leaderWorkerTemplate.size).
// Each pod runs one vLLM/SGLang instance. GPUs per pod = TotalGPUs / PodsSize.
// When PodsSize is 0 or 1, all GPUs are in a single pod.
PodsSize int `json:"pods_size,omitempty"`
// Hardware is the hardware resource allocated to this role.
Hardware HardWare `json:"hardware"`
}
Expand Down
Loading