Skip to content

Commit d3bb1e8

Browse files
committed
fix(vllm-metal): add --runner pooling for embedding models
Signed-off-by: Dorin Geman <dorin.geman@docker.com>
1 parent da4192c commit d3bb1e8

1 file changed

Lines changed: 1 addition & 1 deletion

File tree

pkg/inference/backends/vllmmetal/vllmmetal.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ func (v *vllmMetal) buildArgs(bundle interface{ SafetensorsPath() string }, sock
297297
case inference.BackendModeCompletion:
298298
// Default mode, no additional args needed
299299
case inference.BackendModeEmbedding:
300-
// vllm-metal may handle embedding models automatically
300+
args = append(args, "--runner", "pooling")
301301
case inference.BackendModeReranking:
302302
return nil, fmt.Errorf("reranking mode not supported by vllm-metal backend")
303303
case inference.BackendModeImageGeneration:

0 commit comments

Comments
 (0)