Skip to content

Commit ea46ec9

Browse files
gyliu513gyliu513
committed
address comments from Nir
Signed-off-by: Guangya Liu <gyliu513@gmail.com>
1 parent 2e186f8 commit ea46ec9

4 files changed

Lines changed: 38 additions & 41 deletions

File tree

‎pkg/handlers/request.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -141,25 +141,25 @@ func (s *Server) runRequestPlugins(ctx context.Context, cycleState *plugin.Cycle
141141
defer stageSpan.End()
142142

143143
for _, reqPlugin := range reqPlugins {
144-
name := reqPlugin.TypedName()
144+
typedName := reqPlugin.TypedName()
145145
if verboseEnabled {
146-
verboseLogger.Info("Executing request plugin", "plugin", name)
146+
verboseLogger.Info("Executing request plugin", "plugin", typedName)
147147
}
148-
pluginCtx, span := tracer.Start(ctx, "plugin."+name.Type,
148+
pluginCtx, span := tracer.Start(ctx, "plugin."+typedName.Type,
149149
trace.WithSpanKind(trace.SpanKindInternal),
150150
trace.WithAttributes(
151151
attribute.String("llm_d.plugin.extension_point", requestPluginExtensionPoint),
152-
attribute.String("llm_d.plugin.type", name.Type),
153-
attribute.String("llm_d.plugin.name", name.Name),
152+
attribute.String("llm_d.plugin.type", typedName.Type),
153+
attribute.String("llm_d.plugin.name", typedName.Name),
154154
))
155155
before := time.Now()
156156
err := reqPlugin.ProcessRequest(pluginCtx, cycleState, request)
157-
metrics.RecordPluginProcessingLatency(requestPluginExtensionPoint, name.Type, name.Name, time.Since(before))
157+
metrics.RecordPluginProcessingLatency(requestPluginExtensionPoint, typedName.Type, typedName.Name, time.Since(before))
158158
if err != nil {
159159
span.RecordError(err)
160160
span.SetStatus(codes.Error, err.Error())
161161
span.End()
162-
logger.Error(err, "Failed to execute request plugin", "plugin", name)
162+
logger.Error(err, "Failed to execute request plugin", "plugin", typedName)
163163
return err
164164
}
165165
span.End()

‎pkg/handlers/response.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -161,25 +161,25 @@ func (s *Server) runResponsePlugins(ctx context.Context, cycleState *plugin.Cycl
161161
defer stageSpan.End()
162162

163163
for _, respPlugin := range respPlugins {
164-
name := respPlugin.TypedName()
164+
typedName := respPlugin.TypedName()
165165
if verboseEnabled {
166-
verboseLogger.Info("Executing response plugin", "plugin", name)
166+
verboseLogger.Info("Executing response plugin", "plugin", typedName)
167167
}
168-
pluginCtx, span := tracer.Start(ctx, "plugin."+name.Type,
168+
pluginCtx, span := tracer.Start(ctx, "plugin."+typedName.Type,
169169
trace.WithSpanKind(trace.SpanKindInternal),
170170
trace.WithAttributes(
171171
attribute.String("llm_d.plugin.extension_point", responsePluginExtensionPoint),
172-
attribute.String("llm_d.plugin.type", name.Type),
173-
attribute.String("llm_d.plugin.name", name.Name),
172+
attribute.String("llm_d.plugin.type", typedName.Type),
173+
attribute.String("llm_d.plugin.name", typedName.Name),
174174
))
175175
before := time.Now()
176176
err := respPlugin.ProcessResponse(pluginCtx, cycleState, response)
177-
metrics.RecordPluginProcessingLatency(responsePluginExtensionPoint, name.Type, name.Name, time.Since(before))
177+
metrics.RecordPluginProcessingLatency(responsePluginExtensionPoint, typedName.Type, typedName.Name, time.Since(before))
178178
if err != nil {
179179
span.RecordError(err)
180180
span.SetStatus(codes.Error, err.Error())
181181
span.End()
182-
logger.Error(err, "Failed to execute response plugin", "plugin", name)
182+
logger.Error(err, "Failed to execute response plugin", "plugin", typedName)
183183
return err
184184
}
185185
span.End()

‎pkg/modelselector/model_selector_pipeline.go‎

Lines changed: 23 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -182,29 +182,29 @@ func (p *ModelSelectorPipeline) runFilterPlugins(ctx context.Context, request *r
182182

183183
tracer := tracing.Tracer(modelSelectorTracerScope)
184184
for _, filter := range p.filters {
185-
name := filter.TypedName()
185+
typedName := filter.TypedName()
186186
if verboseEnabled {
187-
verboseLogger.Info("Running filter plugin", "plugin", name)
187+
verboseLogger.Info("Running filter plugin", "plugin", typedName)
188188
}
189-
spanCtx, span := tracer.Start(ctx, "plugin."+name.Type,
189+
spanCtx, span := tracer.Start(ctx, "plugin."+typedName.Type,
190190
trace.WithSpanKind(trace.SpanKindInternal),
191191
trace.WithAttributes(
192192
attribute.String("llm_d.plugin.extension_point", filterExtensionPoint),
193-
attribute.String("llm_d.plugin.type", name.Type),
194-
attribute.String("llm_d.plugin.name", name.Name),
193+
attribute.String("llm_d.plugin.type", typedName.Type),
194+
attribute.String("llm_d.plugin.name", typedName.Name),
195195
attribute.Int("llm_d.filter.candidates_in", len(filteredModels)),
196196
))
197197
before := time.Now()
198198
filteredModels = filter.Filter(spanCtx, cycleState, request, filteredModels)
199-
metrics.RecordPluginProcessingLatency(filterExtensionPoint, name.Type, name.Name, time.Since(before))
199+
metrics.RecordPluginProcessingLatency(filterExtensionPoint, typedName.Type, typedName.Name, time.Since(before))
200200
span.SetAttributes(attribute.Int("llm_d.filter.candidates_out", len(filteredModels)))
201201
span.End()
202202
if debugEnabled {
203-
debugLogger.Info("Completed running filter plugin", "plugin", name, "remainingModels", len(filteredModels))
203+
debugLogger.Info("Completed running filter plugin", "plugin", typedName, "remainingModels", len(filteredModels))
204204
}
205205
if len(filteredModels) == 0 {
206206
if verboseEnabled {
207-
verboseLogger.Info("Filter eliminated all models", "plugin", name)
207+
verboseLogger.Info("Filter eliminated all models", "plugin", typedName)
208208
}
209209
break
210210
}
@@ -236,30 +236,30 @@ func (p *ModelSelectorPipeline) runScorerPlugins(ctx context.Context, request *r
236236

237237
tracer := tracing.Tracer(modelSelectorTracerScope)
238238
for _, scorer := range p.scorers {
239-
name := scorer.TypedName()
239+
typedName := scorer.TypedName()
240240
if verboseEnabled {
241-
verboseLogger.Info("Running scorer plugin", "plugin", name)
241+
verboseLogger.Info("Running scorer plugin", "plugin", typedName)
242242
}
243-
spanCtx, span := tracer.Start(ctx, "plugin."+name.Type,
243+
spanCtx, span := tracer.Start(ctx, "plugin."+typedName.Type,
244244
trace.WithSpanKind(trace.SpanKindInternal),
245245
trace.WithAttributes(
246246
attribute.String("llm_d.plugin.extension_point", scorerExtensionPoint),
247-
attribute.String("llm_d.plugin.type", name.Type),
248-
attribute.String("llm_d.plugin.name", name.Name),
247+
attribute.String("llm_d.plugin.type", typedName.Type),
248+
attribute.String("llm_d.plugin.name", typedName.Name),
249249
attribute.Int("llm_d.scorer.candidate_count", len(models)),
250250
attribute.Float64("llm_d.scorer.weight", scorer.Weight()),
251251
))
252252
before := time.Now()
253253
scores := scorer.Score(spanCtx, cycleState, request, models)
254-
metrics.RecordPluginProcessingLatency(scorerExtensionPoint, name.Type, name.Name, time.Since(before))
254+
metrics.RecordPluginProcessingLatency(scorerExtensionPoint, typedName.Type, typedName.Name, time.Since(before))
255255
span.End()
256256
for model, score := range scores {
257257
if sm, exists := scoredModels[model.GetName()]; exists {
258258
sm.Score += enforceScoreRange(score) * scorer.Weight()
259259
}
260260
}
261261
if debugEnabled {
262-
debugLogger.Info("Completed running scorer plugin", "plugin", name)
262+
debugLogger.Info("Completed running scorer plugin", "plugin", typedName)
263263
}
264264
}
265265
verboseLogger.Info("Completed running scorer plugins")
@@ -284,27 +284,25 @@ func (p *ModelSelectorPipeline) runPickerPlugin(ctx context.Context, cycleState
284284
i++
285285
}
286286

287-
name := p.picker.TypedName()
287+
typedName := p.picker.TypedName()
288288
if verboseEnabled {
289-
verboseLogger.Info("Running picker plugin", "plugin", name)
289+
verboseLogger.Info("Running picker plugin", "plugin", typedName)
290290
}
291-
spanCtx, span := tracing.Tracer(modelSelectorTracerScope).Start(ctx, "plugin."+name.Type,
291+
spanCtx, span := tracing.Tracer(modelSelectorTracerScope).Start(ctx, "plugin."+typedName.Type,
292292
trace.WithSpanKind(trace.SpanKindInternal),
293293
trace.WithAttributes(
294294
attribute.String("llm_d.plugin.extension_point", pickerExtensionPoint),
295-
attribute.String("llm_d.plugin.type", name.Type),
296-
attribute.String("llm_d.plugin.name", name.Name),
295+
attribute.String("llm_d.plugin.type", typedName.Type),
296+
attribute.String("llm_d.plugin.name", typedName.Name),
297297
attribute.Int("llm_d.picker.candidate_count", len(scoredModels)),
298298
))
299299
before := time.Now()
300300
result := p.picker.Pick(spanCtx, cycleState, scoredModels)
301-
metrics.RecordPluginProcessingLatency(pickerExtensionPoint, name.Type, name.Name, time.Since(before))
302-
if result != nil && result.TargetModel != nil {
303-
span.SetAttributes(attribute.String("llm_d.picker.selected_model", result.TargetModel.GetName()))
304-
}
301+
metrics.RecordPluginProcessingLatency(pickerExtensionPoint, typedName.Type, typedName.Name, time.Since(before))
302+
span.SetAttributes(attribute.String("llm_d.picker.selected_model", result.TargetModel.GetName()))
305303
span.End()
306304
if debugEnabled {
307-
debugLogger.Info("Completed running picker plugin", "plugin", name, "result", result)
305+
debugLogger.Info("Completed running picker plugin", "plugin", typedName, "result", result)
308306
}
309307

310308
return result

‎pkg/modelselector/modelselector.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,6 @@ func (s *ModelSelector) Select(ctx context.Context, request *requesthandling.Inf
7474
if err != nil {
7575
span.RecordError(err)
7676
span.SetStatus(codes.Error, err.Error())
77-
} else if result != nil && result.TargetModel != nil {
78-
span.SetAttributes(attribute.String("llm_d.model_selector.selected_model", result.TargetModel.GetName()))
7977
}
8078
}()
8179

@@ -95,6 +93,7 @@ func (s *ModelSelector) Select(ctx context.Context, request *requesthandling.Inf
9593
return nil, err
9694
}
9795

96+
span.SetAttributes(attribute.String("llm_d.model_selector.selected_model", result.TargetModel.GetName()))
9897
logger.V(logutil.VERBOSE).Info("Model selection completed", "selectedModel", result.TargetModel.GetName())
9998

10099
return result, nil

0 commit comments

Comments
 (0)