|
1 | 1 | package application |
2 | 2 |
|
3 | 3 | import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + |
4 | 7 | "github.com/mudler/LocalAI/core/backend" |
5 | 8 | "github.com/mudler/LocalAI/core/config" |
6 | 9 | ) |
7 | 10 |
|
8 | | -// adapterConfig resolves a model name to its runtime ModelConfig, or |
9 | | -// nil when the name is unknown. Shared by the router-facing factories |
10 | | -// below and by ModelConfigLookup. |
| 11 | +// adapterConfig resolves a model name to its runtime ModelConfig, or nil when |
| 12 | +// unknown. LoadModelConfigFileByNameDefaultOptions never returns nil — for an |
| 13 | +// unknown name it returns a defaults-filled stub with an empty Name (the YAML |
| 14 | +// `name:` field is required by Validate), which is how we tell the two apart. |
11 | 15 | func (a *Application) adapterConfig(modelName string) *config.ModelConfig { |
12 | 16 | cfg, err := a.backendLoader.LoadModelConfigFileByNameDefaultOptions(modelName, a.applicationConfig) |
13 | | - if err != nil || cfg == nil { |
| 17 | + if err != nil || cfg == nil || cfg.Name == "" { |
14 | 18 | return nil |
15 | 19 | } |
16 | 20 | return cfg |
17 | 21 | } |
18 | 22 |
|
19 | | -// ModelConfigLookup is the lookup function the router middleware's |
20 | | -// classifier validator uses to confirm classifier_model declares |
21 | | -// FLAG_SCORE before binding it. |
| 23 | +// ModelConfigLookup is the lookup the router middleware's classifier validator |
| 24 | +// uses to confirm classifier_model declares FLAG_SCORE before binding it. |
22 | 25 | func (a *Application) ModelConfigLookup() func(modelName string) *config.ModelConfig { |
23 | 26 | return a.adapterConfig |
24 | 27 | } |
25 | 28 |
|
26 | | -// Scorer returns a backend.Scorer bound to the named model, or nil |
27 | | -// when the model is unknown. Used as a method value (app.Scorer) by |
28 | | -// router.ClassifierDeps — no factory-of-factory wrapper needed. |
| 29 | +// The router-facing factories below (Scorer, Embedder, Reranker, TokenCounter) |
| 30 | +// bind a model NAME at construction and re-resolve the CONFIG on every call. |
| 31 | +// Capturing the config at construction would bake in whatever state |
| 32 | +// adapterConfig saw first — including a stub returned before the YAML reached |
| 33 | +// bcl.configs (e.g. /import-model or gallery install racing startup). The |
| 34 | +// classifier registry caches factories by router-config fingerprint, so a |
| 35 | +// once-stale capture stays stale until the router config is edited. |
| 36 | + |
29 | 37 | func (a *Application) Scorer(modelName string) backend.Scorer { |
30 | | - cfg := a.adapterConfig(modelName) |
| 38 | + if a.adapterConfig(modelName) == nil { |
| 39 | + return nil |
| 40 | + } |
| 41 | + return &lazyScorer{app: a, modelName: modelName} |
| 42 | +} |
| 43 | + |
| 44 | +type lazyScorer struct { |
| 45 | + app *Application |
| 46 | + modelName string |
| 47 | +} |
| 48 | + |
| 49 | +func (l *lazyScorer) Score(ctx context.Context, prompt string, candidates []string) ([]backend.CandidateScore, error) { |
| 50 | + cfg := l.app.adapterConfig(l.modelName) |
31 | 51 | if cfg == nil { |
| 52 | + return nil, fmt.Errorf("scorer: model %q no longer available", l.modelName) |
| 53 | + } |
| 54 | + return backend.NewScorer(l.app.modelLoader, *cfg, l.app.applicationConfig).Score(ctx, prompt, candidates) |
| 55 | +} |
| 56 | + |
| 57 | +// TokenCounter returns a func so the middleware's literal field type accepts |
| 58 | +// it as a method value without importing core/http/middleware from here. |
| 59 | +func (a *Application) TokenCounter(modelName string) func(string) (int, error) { |
| 60 | + if a.adapterConfig(modelName) == nil { |
32 | 61 | return nil |
33 | 62 | } |
34 | | - return backend.NewScorer(a.modelLoader, *cfg, a.applicationConfig) |
| 63 | + return func(text string) (int, error) { |
| 64 | + cfg := a.adapterConfig(modelName) |
| 65 | + if cfg == nil { |
| 66 | + return 0, fmt.Errorf("token counter: model %q no longer available", modelName) |
| 67 | + } |
| 68 | + resp, err := backend.ModelTokenize(text, a.modelLoader, *cfg, a.applicationConfig) |
| 69 | + if err != nil { |
| 70 | + return 0, err |
| 71 | + } |
| 72 | + return len(resp.Tokens), nil |
| 73 | + } |
35 | 74 | } |
36 | 75 |
|
37 | | -// Reranker returns a backend.Reranker bound to the named model, or |
38 | | -// nil when unknown. The reranker model's `type:` (e.g. "colbert") |
39 | | -// selects the scoring head inside the rerankers backend. |
40 | 76 | func (a *Application) Reranker(modelName string) backend.Reranker { |
41 | | - cfg := a.adapterConfig(modelName) |
42 | | - if cfg == nil { |
| 77 | + if a.adapterConfig(modelName) == nil { |
43 | 78 | return nil |
44 | 79 | } |
45 | | - return backend.NewReranker(a.modelLoader, *cfg, a.applicationConfig) |
| 80 | + return &lazyReranker{app: a, modelName: modelName} |
46 | 81 | } |
47 | 82 |
|
48 | | -// Embedder returns a backend.Embedder bound to the named model, or |
49 | | -// nil when unknown. Used by the router's L2 embedding cache. |
50 | | -func (a *Application) Embedder(modelName string) backend.Embedder { |
51 | | - cfg := a.adapterConfig(modelName) |
| 83 | +type lazyReranker struct { |
| 84 | + app *Application |
| 85 | + modelName string |
| 86 | +} |
| 87 | + |
| 88 | +func (l *lazyReranker) Rerank(ctx context.Context, query string, documents []string) ([]backend.RerankResult, error) { |
| 89 | + cfg := l.app.adapterConfig(l.modelName) |
52 | 90 | if cfg == nil { |
| 91 | + return nil, fmt.Errorf("reranker: model %q no longer available", l.modelName) |
| 92 | + } |
| 93 | + return backend.NewReranker(l.app.modelLoader, *cfg, l.app.applicationConfig).Rerank(ctx, query, documents) |
| 94 | +} |
| 95 | + |
| 96 | +func (a *Application) Embedder(modelName string) backend.Embedder { |
| 97 | + if a.adapterConfig(modelName) == nil { |
53 | 98 | return nil |
54 | 99 | } |
55 | | - return backend.NewEmbedder(a.modelLoader, *cfg, a.applicationConfig) |
| 100 | + return &lazyEmbedder{app: a, modelName: modelName} |
| 101 | +} |
| 102 | + |
| 103 | +type lazyEmbedder struct { |
| 104 | + app *Application |
| 105 | + modelName string |
| 106 | +} |
| 107 | + |
| 108 | +func (l *lazyEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { |
| 109 | + cfg := l.app.adapterConfig(l.modelName) |
| 110 | + if cfg == nil { |
| 111 | + return nil, fmt.Errorf("embedder: model %q no longer available", l.modelName) |
| 112 | + } |
| 113 | + return backend.NewEmbedder(l.app.modelLoader, *cfg, l.app.applicationConfig).Embed(ctx, text) |
56 | 114 | } |
57 | 115 |
|
58 | | -// VectorStore returns a backend.VectorStore for the named collection, |
59 | | -// or nil when the name is empty. Each router model gets its own |
60 | | -// backend process via the model loader's cache keyed by storeName. |
| 116 | +// VectorStore takes a store name, not a model name — no adapterConfig, no |
| 117 | +// staleness to avoid. |
61 | 118 | func (a *Application) VectorStore(storeName string) backend.VectorStore { |
62 | 119 | return backend.NewVectorStore(a.modelLoader, a.applicationConfig, storeName) |
63 | 120 | } |
0 commit comments