Skip to content

[Refactor] Scope-first runtime tie-breaking; deprecate Weight #603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
YouNeedCryDear wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[Refactor] Scope-first runtime tie-breaking; deprecate Weight #603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2b3083a
deprecate weight since matcher requires strict equality on model form…
YouNeedCryDear May 8, 2026
f98558b
enforce scope before size approximity
YouNeedCryDear May 8, 2026
ab21765
deprecate weight and enforce namespace scoped overwrite
YouNeedCryDear May 8, 2026
4855ad9
update serving runtime webhook to follow scope priority size and name
YouNeedCryDear May 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkg/runtimeselector/matcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ func (m *DefaultRuntimeMatcher) evaluateFormatMatch(model *v1beta1.BaseModelSpec
match.FormatMatch = false
}

// TODO: stop populating match.Weight; the field is deprecated and no longer read by the scorer.
if match.FormatMatch && format.ModelFormat.Weight > 0 {
match.Weight += format.ModelFormat.Weight * int64(match.Priority)
}
Expand All @@ -245,6 +246,7 @@ func (m *DefaultRuntimeMatcher) evaluateFormatMatch(model *v1beta1.BaseModelSpec
match.FrameworkMatch = false
}

// TODO: stop populating match.Weight; the field is deprecated and no longer read by the scorer.
if match.FrameworkMatch && format.ModelFramework.Weight > 0 {
match.Weight += format.ModelFramework.Weight * int64(match.Priority)
}
Expand Down
84 changes: 38 additions & 46 deletions pkg/runtimeselector/scorer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,9 @@ func NewDefaultRuntimeScorer(config *Config) RuntimeScorer {
}
}

// CalculateScore returns a score for how well a runtime matches a model.
// The score is calculated based on:
// 1. Model format match and weight
// 2. Model framework match and weight
// 3. Priority multiplier
// 4. Model size proximity (if applicable)
// CalculateScore returns the priority of the best matching auto-select format.
// Strict format/framework compatibility is enforced separately by the matcher,
// so a non-zero score here simply means "this runtime matches at this priority".
func (s *DefaultRuntimeScorer) CalculateScore(runtime *v1beta1.ServingRuntimeSpec, model *v1beta1.BaseModelSpec) (int64, error) {
ctx := context.Background()
logger := log.FromContext(ctx)
Expand All @@ -35,8 +32,7 @@ func (s *DefaultRuntimeScorer) CalculateScore(runtime *v1beta1.ServingRuntimeSpe

// Go through all supported model formats in runtime
for _, supportedFormat := range runtime.SupportedModelFormats {
// Skip if autoSelect is explicitly false
if supportedFormat.AutoSelect != nil && !(*supportedFormat.AutoSelect) {
if !supportedFormat.IsAutoSelectEnabled() {
continue
}

Expand Down Expand Up @@ -64,13 +60,27 @@ func (s *DefaultRuntimeScorer) CalculateScore(runtime *v1beta1.ServingRuntimeSpe

// CompareRuntimes compares two runtime matches for a given model.
// Returns positive if r1 is better, negative if r2 is better, 0 if equal.
//
// Tie-breaking order (strict):
// 1. Scope (namespace-scoped beats cluster-scoped)
// 2. Priority (higher wins)
// 3. Model size proximity (closer to the model size wins).
// 4. Name (alphabetical, deterministic).
func (s *DefaultRuntimeScorer) CompareRuntimes(r1, r2 RuntimeMatch, model *v1beta1.BaseModelSpec) int {
// First, compare by score
// Prefer namespace-scoped runtimes over cluster-scoped
if r1.IsCluster != r2.IsCluster {
if r1.IsCluster {
return -1 // r2 is namespace-scoped, prefer it
}
return 1 // r1 is namespace-scoped, prefer it
}

// Within the same scope, compare by priority (Score is the priority).
if r1.Score != r2.Score {
return int(r1.Score - r2.Score)
}

// If scores are equal, compare by model size range if available
// If still equal, compare by model size range if available
if model.ModelParameterSize != nil {
r1SizeScore := s.calculateSizeScore(r1, model)
r2SizeScore := s.calculateSizeScore(r2, model)
Expand All @@ -81,14 +91,6 @@ func (s *DefaultRuntimeScorer) CompareRuntimes(r1, r2 RuntimeMatch, model *v1bet
}
}

// If still equal, prefer namespace-scoped runtimes over cluster-scoped
if r1.IsCluster != r2.IsCluster {
if r1.IsCluster {
return -1 // r2 is namespace-scoped, prefer it
}
return 1 // r1 is namespace-scoped, prefer it
}

// Finally, compare by name for deterministic ordering
if r1.Name < r2.Name {
return 1
Expand All @@ -99,23 +101,27 @@ func (s *DefaultRuntimeScorer) CompareRuntimes(r1, r2 RuntimeMatch, model *v1bet
return 0
}

// calculateFormatScore calculates the score for a specific supported format.
// This matches the exact logic from the original score() function.
// CalculateFormatScore returns the format's Priority when the model is
// compatible with the supported format, and 0 otherwise. Weight is intentionally
// ignored: matcher.go already enforces strict compatibility, so Weight only
// produced scoring collisions.
func (s *DefaultRuntimeScorer) CalculateFormatScore(model *v1beta1.BaseModelSpec, supportedFormat v1beta1.SupportedModelFormat, priority int64) int64 {
// Compare model format
modelFormatMatches := false
if supportedFormat.ModelFormat != nil {
if supportedFormat.ModelFormat.Name != model.ModelFormat.Name {
return 0 // Format name doesn't match
}
// Compare versions if both are specified
if supportedFormat.ModelFormat.Version != nil && model.ModelFormat.Version != nil {
switch {
case supportedFormat.ModelFormat.Version != nil && model.ModelFormat.Version != nil:
modelFormatMatches = s.compareVersions(supportedFormat.ModelFormat, &model.ModelFormat)
if !modelFormatMatches {
return 0 // Version doesn't match
}
} else {
case supportedFormat.ModelFormat.Version == nil && model.ModelFormat.Version == nil:
modelFormatMatches = true
default:
// Version asymmetry: matcher rejects, so we must too.
return 0
}
}

Expand All @@ -125,39 +131,25 @@ func (s *DefaultRuntimeScorer) CalculateFormatScore(model *v1beta1.BaseModelSpec
if supportedFormat.ModelFramework.Name != model.ModelFramework.Name {
return 0 // Framework name doesn't match
}
// Compare versions if both are specified
if supportedFormat.ModelFramework.Version != nil && model.ModelFramework.Version != nil {
switch {
case supportedFormat.ModelFramework.Version != nil && model.ModelFramework.Version != nil:
modelFrameworkMatches = s.compareFrameworkVersions(supportedFormat.ModelFramework, model.ModelFramework)
if !modelFrameworkMatches {
return 0 // Version doesn't match
}
} else {
case supportedFormat.ModelFramework.Version == nil && model.ModelFramework.Version == nil:
modelFrameworkMatches = true
default:
return 0
}
}

// Check the matching condition (same as original line 223-224)
if (modelFormatMatches || supportedFormat.ModelFormat == nil) &&
(modelFrameworkMatches || (supportedFormat.ModelFramework == nil && model.ModelFramework == nil)) {

// Calculate weighted score
var currentScore int64 = 0
if modelFormatMatches && supportedFormat.ModelFormat != nil {
weight := supportedFormat.ModelFormat.Weight
if weight == 0 {
weight = s.config.ModelFormatWeight
}
currentScore += weight * priority
}
if modelFrameworkMatches && supportedFormat.ModelFramework != nil {
weight := supportedFormat.ModelFramework.Weight
if weight == 0 {
weight = s.config.ModelFrameworkWeight
}
currentScore += weight * priority
// Require at least one positive match
if modelFormatMatches || modelFrameworkMatches {
return priority
}

return currentScore
}

return 0
Expand Down
22 changes: 8 additions & 14 deletions pkg/runtimeselector/selector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,29 +115,21 @@ func (s *defaultSelector) GetCompatibleRuntimes(ctx context.Context, model *v1be
return nil, fmt.Errorf("failed to fetch runtimes: %w", err)
}

var namespaceMatches []RuntimeMatch
var clusterMatches []RuntimeMatch

// Process namespace-scoped runtimes
var matches []RuntimeMatch
for _, runtime := range collection.NamespaceRuntimes {
if match := s.evaluateRuntime(ctx, &runtime.Spec, model, isvc, runtime.Name, false); match != nil {
namespaceMatches = append(namespaceMatches, *match)
matches = append(matches, *match)
}
}

// Process cluster-scoped runtimes
for _, runtime := range collection.ClusterRuntimes {
if match := s.evaluateRuntime(ctx, &runtime.Spec, model, isvc, runtime.Name, true); match != nil {
clusterMatches = append(clusterMatches, *match)
matches = append(matches, *match)
}
}

// Sort namespace and cluster matches separately
s.sortMatches(namespaceMatches, model)
s.sortMatches(clusterMatches, model)

// Append cluster matches after namespace matches (namespace-scoped have priority)
matches := append(namespaceMatches, clusterMatches...)
// Sort globally
// CompareRuntimes orders by scope -> priority -> size -> name.
s.sortMatches(matches, model)

logger.Info("Found compatible runtimes",
"model", model.ModelFormat.Name,
Expand Down Expand Up @@ -321,6 +313,8 @@ func getModelName(model *v1beta1.BaseModelSpec) string {
// userSpecifiedRuntime indicates whether the runtime is a user-selected runtime or an automatically selected runtime
// if userSpecifiedRuntime is true, the function will consider all supportedModelFormats in the runtime
// if userSpecifiedRuntime is false, the function will only consider supportedModelFormats with autoSelect enabled
//
// TODO: pass the format's own Priority (defaulting to DefaultPriority when nil) instead of DefaultPriority for every call, so the highest-priority matching format wins under priority-only scoring.
func (s *defaultSelector) GetSupportedModelFormat(ctx context.Context, runtime *v1beta1.ServingRuntimeSpec, model *v1beta1.BaseModelSpec, userSpecifiedRuntime bool) *v1beta1.SupportedModelFormat {
if runtime.SupportedModelFormats == nil {
return nil
Expand Down
Loading
Loading