Skip to content

Commit 3205ffe

Browse files
committed
fix: eliminate empty tier labels in selection metrics recording
Signed-off-by: Senan Zedan <szedan@redhat.com>
1 parent ca224e9 commit 3205ffe

4 files changed

Lines changed: 17 additions & 44 deletions

File tree

src/semantic-router/pkg/extproc/req_filter_classification.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,7 @@ func (r *OpenAIRouter) selectModelFromCandidates(modelRefs []config.ModelRef, de
116116
modelRefs[i].LoRAName == result.SelectedModel {
117117
logging.Infof("[ModelSelection] Selected %s (method=%s, score=%.4f, confidence=%.2f): %s",
118118
result.SelectedModel, method, result.Score, result.Confidence, result.Reasoning)
119-
// Record selection metrics
120-
selection.RecordSelection(string(method), decisionName, result.SelectedModel, result.Score)
119+
selection.RecordSelection(string(method), decisionName, result.SelectedModel, result.Tier, result.Score)
121120
return &modelRefs[i], string(method)
122121
}
123122
}

src/semantic-router/pkg/selection/metrics.go

Lines changed: 9 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -400,32 +400,18 @@ func IsMetricsEnabled() bool {
400400
return metricsEnabled
401401
}
402402

403-
// RecordSelection records a basic model selection event
404-
func RecordSelection(method string, decision string, model string, score float64) {
403+
// RecordSelection records a basic model selection event with tier label.
404+
func RecordSelection(method string, decision string, model string, tier AlgorithmTier, score float64) {
405405
if !metricsEnabled {
406406
return
407407
}
408408

409-
ModelSelectionTotal.WithLabelValues(method, model, decision, "").Inc()
409+
tierStr := string(tier)
410+
ModelSelectionTotal.WithLabelValues(method, model, decision, tierStr).Inc()
410411
ModelSelectionScore.WithLabelValues(method, model).Observe(score)
411412
ModelSelectionHistory.WithLabelValues(method, decision).Inc()
412413
}
413414

414-
// RecordSelectionFull records a model selection event with all metrics
415-
func RecordSelectionFull(method SelectionMethod, model string, decision string, score, confidence float64, duration time.Duration) {
416-
if !metricsEnabled {
417-
return
418-
}
419-
420-
methodStr := string(method)
421-
422-
ModelSelectionTotal.WithLabelValues(methodStr, model, decision, "").Inc()
423-
ModelSelectionDuration.WithLabelValues(methodStr, "").Observe(duration.Seconds())
424-
ModelSelectionScore.WithLabelValues(methodStr, model).Observe(score)
425-
ModelSelectionConfidence.WithLabelValues(methodStr, "").Observe(confidence)
426-
ModelSelectionHistory.WithLabelValues(methodStr, decision).Inc()
427-
}
428-
429415
// RecordEloRating records the current Elo rating for a model in a category
430416
func RecordEloRating(model, category string, rating float64) {
431417
if !metricsEnabled {
@@ -608,15 +594,13 @@ func calculateAgreementRatio(choices []string) float64 {
608594
}
609595

610596
// RecordHybridSelection records metrics for a hybrid selection including component agreement
611-
func RecordHybridSelection(selectedModel string, decision string, componentChoices map[string]string, score, confidence float64, duration time.Duration) {
597+
func RecordHybridSelection(selectedModel string, decision string, componentChoices map[string]string, tier AlgorithmTier, score, confidence float64, duration time.Duration) {
612598
if !metricsEnabled {
613599
return
614600
}
615601

616-
// Record standard selection metrics
617-
RecordSelectionFull(MethodHybrid, selectedModel, decision, score, confidence, duration)
602+
RecordSelectionWithTier(MethodHybrid, selectedModel, decision, tier, score, confidence, duration)
618603

619-
// Calculate and record component agreement
620604
if len(componentChoices) > 1 {
621605
choices := make([]string, 0, len(componentChoices))
622606
for _, model := range componentChoices {
@@ -677,7 +661,7 @@ func RecordRouterDCAffinity(model string, affinity float64) {
677661
// --- RL-Driven metrics recording functions ---
678662

679663
// RecordRLSelection records a RL-driven model selection event
680-
func RecordRLSelection(model, category, userID string, score float64) {
664+
func RecordRLSelection(model, category, userID string, tier AlgorithmTier, score float64) {
681665
if !metricsEnabled {
682666
return
683667
}
@@ -686,11 +670,11 @@ func RecordRLSelection(model, category, userID string, score float64) {
686670
category = "_global"
687671
}
688672

689-
ModelSelectionTotal.WithLabelValues("rl_driven", model, category, "experimental").Inc()
673+
tierStr := string(tier)
674+
ModelSelectionTotal.WithLabelValues("rl_driven", model, category, tierStr).Inc()
690675
ModelSelectionScore.WithLabelValues("rl_driven", model).Observe(score)
691676
ModelSelectionHistory.WithLabelValues("rl_driven", category).Inc()
692677

693-
// Track personalized selections
694678
if userID != "" {
695679
RLDrivenPersonalizedSelections.WithLabelValues(model, category).Inc()
696680
}

src/semantic-router/pkg/selection/metrics_test.go

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -70,18 +70,10 @@ func TestRecordSelection(t *testing.T) {
7070
InitializeMetrics()
7171

7272
// Should not panic
73-
RecordSelection("elo", "tech", "llama3.2:3b", 0.85)
74-
RecordSelection("router_dc", "finance", "phi4", 0.72)
75-
RecordSelection("hybrid", "general", "gemma3:27b", 0.91)
76-
}
77-
78-
func TestRecordSelectionFull(t *testing.T) {
79-
InitializeMetrics()
80-
81-
// Should not panic
82-
RecordSelectionFull(MethodElo, "llama3.2:3b", "tech", 0.85, 0.92, 5*time.Millisecond)
83-
RecordSelectionFull(MethodRouterDC, "phi4", "finance", 0.72, 0.88, 3*time.Millisecond)
84-
RecordSelectionFull(MethodHybrid, "gemma3:27b", "general", 0.91, 0.95, 10*time.Millisecond)
73+
RecordSelection("elo", "tech", "llama3.2:3b", TierSupported, 0.85)
74+
RecordSelection("router_dc", "finance", "phi4", TierSupported, 0.72)
75+
RecordSelection("hybrid", "general", "gemma3:27b", TierSupported, 0.91)
76+
RecordSelection("automix", "general", "gemma3:27b", TierExperimental, 0.91)
8577
}
8678

8779
func TestRecordEloRating(t *testing.T) {
@@ -200,7 +192,7 @@ func TestRecordHybridSelection(t *testing.T) {
200192
}
201193

202194
// Should not panic
203-
RecordHybridSelection("llama3.2:3b", "tech", componentChoices, 0.85, 0.92, 10*time.Millisecond)
195+
RecordHybridSelection("llama3.2:3b", "tech", componentChoices, TierSupported, 0.85, 0.92, 10*time.Millisecond)
204196
}
205197

206198
func TestCalculateAgreementRatio(t *testing.T) {

src/semantic-router/pkg/selection/rl_driven.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -600,8 +600,7 @@ func (r *RLDrivenSelector) Select(ctx context.Context, selCtx *SelectionContext)
600600
logging.Infof("[RLDrivenSelector] Selected %s (score=%.4f, confidence=%.2f)",
601601
selectedModel.Model, selectedScore, confidence)
602602

603-
// Record metrics
604-
RecordRLSelection(selectedModel.Model, selCtx.DecisionName, selCtx.UserID, selectedScore)
603+
RecordRLSelection(selectedModel.Model, selCtx.DecisionName, selCtx.UserID, r.Tier(), selectedScore)
605604

606605
return &SelectionResult{
607606
SelectedModel: selectedModel.Model,
@@ -675,8 +674,7 @@ func (r *RLDrivenSelector) selectWithRouterR1(ctx context.Context, selCtx *Selec
675674
logging.Infof("[RLDrivenSelector] Router-R1 selected %s (thinking: %s)",
676675
selectedModel.Model, truncateString(response.Thinking, 100))
677676

678-
// Record metrics
679-
RecordRLSelection(selectedModel.Model, selCtx.DecisionName, selCtx.UserID, 1.0)
677+
RecordRLSelection(selectedModel.Model, selCtx.DecisionName, selCtx.UserID, r.Tier(), 1.0)
680678

681679
return &SelectionResult{
682680
SelectedModel: selectedModel.Model,

0 commit comments

Comments
 (0)