Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions core/application/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -553,12 +553,17 @@ func (a *Application) start() error {
// once at startup and reused across chat sessions that opt in via metadata.
if !a.applicationConfig.DisableLocalAIAssistant {
holder := mcpTools.NewLocalAIAssistantHolder()
var nodeRegistry *nodes.NodeRegistry
if a.distributed != nil {
nodeRegistry = a.distributed.Registry
}
assistantClient := localaiInproc.New(
a.applicationConfig,
a.applicationConfig.SystemState,
a.backendLoader,
a.modelLoader,
a.galleryService,
nodeRegistry,
)
// Wire usage tracking so the assistant's get_usage_stats tool
// returns real data; nil values keep the tool returning a clear
Expand Down
4 changes: 4 additions & 0 deletions pkg/mcp/localaitools/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ type LocalAIClient interface {
// ---- System ----
SystemInfo(ctx context.Context) (*SystemInfo, error)
ListNodes(ctx context.Context) ([]Node, error)
ListScheduling(ctx context.Context) ([]ModelSchedulingConfig, error)
GetScheduling(ctx context.Context, modelName string) (*ModelSchedulingConfig, error)
SetScheduling(ctx context.Context, req SetSchedulingRequest) (*ModelSchedulingConfig, error)
DeleteScheduling(ctx context.Context, modelName string) error
// SetNodeVRAMBudget sets (or, with an empty budget, clears) a federated
// node's VRAM allocation cap as a sticky admin override. Only meaningful
// in distributed mode; single-process clients report it as unavailable.
Expand Down
4 changes: 4 additions & 0 deletions pkg/mcp/localaitools/coverage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ var toolToHTTPRoute = map[string]string{
ToolListKnownBackends: "GET /backends/known",
ToolSystemInfo: "GET / (welcome JSON)",
ToolListNodes: "GET /api/nodes",
ToolListScheduling: "GET /api/nodes/scheduling",
ToolGetScheduling: "GET /api/nodes/scheduling/:model",
ToolVRAMEstimate: "POST /api/models/vram-estimate",
ToolGetBranding: "GET /api/branding",
ToolGetUsageStats: "GET /api/usage (or /api/usage/all when all=true)",
Expand All @@ -60,6 +62,8 @@ var toolToHTTPRoute = map[string]string{
ToolCreateVoiceProfile: "POST /api/voice-profiles",
ToolDeleteVoiceProfile: "DELETE /api/voice-profiles/:id",
ToolSetNodeVRAMBudget: "PUT /api/nodes/:id/vram-budget",
ToolSetScheduling: "POST /api/nodes/scheduling",
ToolDeleteScheduling: "DELETE /api/nodes/scheduling/:model",
}

// allKnownTools is the union of expectedFullCatalog (defined in
Expand Down
35 changes: 35 additions & 0 deletions pkg/mcp/localaitools/dto.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,41 @@ type SetNodeVRAMBudgetRequest struct {
Budget string `json:"budget,omitempty" jsonschema:"VRAM allocation cap as a percentage (e.g. 80%) or absolute amount (e.g. 12GB). Empty string clears the override."`
}

// ModelSchedulingConfig is the MCP wire shape for one per-model distributed
// scheduling rule. Keep this DTO explicit instead of aliasing the node-registry
// model so the MCP contract only exposes operator-facing scheduling fields.
type ModelSchedulingConfig struct {
ModelName string `json:"model_name"`
NodeSelector string `json:"node_selector,omitempty"`
MinReplicas int `json:"min_replicas"`
MaxReplicas int `json:"max_replicas"`
SpreadAll bool `json:"spread_all,omitempty"`
RoutePolicy string `json:"route_policy,omitempty"`
BalanceAbsThreshold int `json:"balance_abs_threshold,omitempty"`
BalanceRelThreshold float64 `json:"balance_rel_threshold,omitempty"`
MinPrefixMatch float64 `json:"min_prefix_match,omitempty"`
}

// SetSchedulingRequest is the input for set_scheduling. It mirrors
// /api/nodes/scheduling so standalone MCP and REST callers preserve the same
// PATCH-style semantics for the optional prefix-cache routing fields.
type SetSchedulingRequest struct {
ModelName string `json:"model_name" jsonschema:"Installed model name whose distributed scheduling rule should be created or updated."`
NodeSelector map[string]string `json:"node_selector,omitempty" jsonschema:"Optional node-label selector. Empty means any healthy backend node."`
MinReplicas int `json:"min_replicas" jsonschema:"Minimum desired replicas. Mutually exclusive with spread_all."`
MaxReplicas int `json:"max_replicas" jsonschema:"Maximum desired replicas. Must be >= min_replicas when non-zero. Mutually exclusive with spread_all."`
SpreadAll bool `json:"spread_all,omitempty" jsonschema:"When true, keep one replica on every matching node. Mutually exclusive with min_replicas/max_replicas."`
RoutePolicy *string `json:"route_policy,omitempty" jsonschema:"Optional prefix-cache route policy override. Omit to preserve the existing value on updates."`
BalanceAbsThreshold *int `json:"balance_abs_threshold,omitempty" jsonschema:"Optional absolute imbalance threshold override. Omit to preserve the existing value on updates."`
BalanceRelThreshold *float64 `json:"balance_rel_threshold,omitempty" jsonschema:"Optional relative imbalance threshold override. Omit to preserve the existing value on updates."`
MinPrefixMatch *float64 `json:"min_prefix_match,omitempty" jsonschema:"Optional minimum prefix match threshold override. Omit to preserve the existing value on updates."`
}

// DeleteSchedulingRequest identifies the model scheduling rule to remove.
type DeleteSchedulingRequest struct {
ModelName string `json:"model_name" jsonschema:"Installed model name whose scheduling config should be removed."`
}

// ImportModelURIRequest is the input for import_model_uri. It mirrors the
// REST surface (`/models/import-uri`) closely so both clients can produce
// identical responses; the BackendPreference is a flat field rather than the
Expand Down
36 changes: 36 additions & 0 deletions pkg/mcp/localaitools/fakes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ type fakeClient struct {
upgradeBackend func(string) (string, error)
systemInfo func() (*SystemInfo, error)
listNodes func() ([]Node, error)
listScheduling func() ([]ModelSchedulingConfig, error)
getScheduling func(string) (*ModelSchedulingConfig, error)
setScheduling func(SetSchedulingRequest) (*ModelSchedulingConfig, error)
deleteScheduling func(string) error
setNodeVRAMBudget func(string, string) error
vramEstimate func(VRAMEstimateRequest) (*vram.EstimateResult, error)
toggleModelState func(string, modeladmin.Action) error
Expand Down Expand Up @@ -230,6 +234,38 @@ func (f *fakeClient) ListNodes(_ context.Context) ([]Node, error) {
return nil, nil
}

func (f *fakeClient) ListScheduling(_ context.Context) ([]ModelSchedulingConfig, error) {
f.record("ListScheduling", nil)
if f.listScheduling != nil {
return f.listScheduling()
}
return []ModelSchedulingConfig{}, nil
}

func (f *fakeClient) GetScheduling(_ context.Context, modelName string) (*ModelSchedulingConfig, error) {
f.record("GetScheduling", modelName)
if f.getScheduling != nil {
return f.getScheduling(modelName)
}
return &ModelSchedulingConfig{ModelName: modelName}, nil
}

func (f *fakeClient) SetScheduling(_ context.Context, req SetSchedulingRequest) (*ModelSchedulingConfig, error) {
f.record("SetScheduling", req)
if f.setScheduling != nil {
return f.setScheduling(req)
}
return &ModelSchedulingConfig{ModelName: req.ModelName, MinReplicas: req.MinReplicas, MaxReplicas: req.MaxReplicas, SpreadAll: req.SpreadAll}, nil
}

func (f *fakeClient) DeleteScheduling(_ context.Context, modelName string) error {
f.record("DeleteScheduling", modelName)
if f.deleteScheduling != nil {
return f.deleteScheduling(modelName)
}
return nil
}

func (f *fakeClient) SetNodeVRAMBudget(_ context.Context, nodeID, budget string) error {
f.record("SetNodeVRAMBudget", []any{nodeID, budget})
if f.setNodeVRAMBudget != nil {
Expand Down
43 changes: 43 additions & 0 deletions pkg/mcp/localaitools/httpapi/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,49 @@ func (c *Client) ListNodes(ctx context.Context) ([]localaitools.Node, error) {
return out, nil
}

func (c *Client) ListScheduling(ctx context.Context) ([]localaitools.ModelSchedulingConfig, error) {
var out []localaitools.ModelSchedulingConfig
if err := c.do(ctx, http.MethodGet, routeScheduling, nil, &out); err != nil {
if errors.Is(err, ErrHTTPNotFound) {
return []localaitools.ModelSchedulingConfig{}, nil
}
return nil, err
}
return out, nil
}

func (c *Client) GetScheduling(ctx context.Context, modelName string) (*localaitools.ModelSchedulingConfig, error) {
if modelName == "" {
return nil, errors.New("model_name is required")
}
var out localaitools.ModelSchedulingConfig
if err := c.do(ctx, http.MethodGet, routeModelScheduling(modelName), nil, &out); err != nil {
if errors.Is(err, ErrHTTPNotFound) {
return nil, nil
}
return nil, err
}
return &out, nil
}

func (c *Client) SetScheduling(ctx context.Context, req localaitools.SetSchedulingRequest) (*localaitools.ModelSchedulingConfig, error) {
if req.ModelName == "" {
return nil, errors.New("model_name is required")
}
var out localaitools.ModelSchedulingConfig
if err := c.do(ctx, http.MethodPost, routeScheduling, req, &out); err != nil {
return nil, err
}
return &out, nil
}

func (c *Client) DeleteScheduling(ctx context.Context, modelName string) error {
if modelName == "" {
return errors.New("model_name is required")
}
return c.do(ctx, http.MethodDelete, routeModelScheduling(modelName), nil, nil)
}

func (c *Client) SetNodeVRAMBudget(ctx context.Context, nodeID, budget string) error {
// PUT with an empty value clears the override server-side (Task 9), so we
// use PUT uniformly rather than switching to DELETE for the clear case.
Expand Down
80 changes: 80 additions & 0 deletions pkg/mcp/localaitools/httpapi/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,43 @@ func fakeLocalAI() *httptest.Server {
})
})

mux.HandleFunc("/api/nodes/scheduling", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
_ = json.NewEncoder(w).Encode([]map[string]any{{
"id": "sched-1",
"model_name": "qwen",
"min_replicas": 1,
"max_replicas": 2,
"unsatisfiable_ticks": 1,
"unsatisfiable_until": "2026-01-01T00:00:00Z",
"created_at": "2026-01-01T00:00:00Z",
"updated_at": "2026-01-01T00:00:00Z",
}})
case http.MethodPost:
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
body["id"] = "sched-1"
_ = json.NewEncoder(w).Encode(body)
default:
http.Error(w, "method", http.StatusMethodNotAllowed)
}
})

mux.HandleFunc("/api/nodes/scheduling/qwen", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
_ = json.NewEncoder(w).Encode(map[string]any{"model_name": "qwen", "spread_all": true})
case http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
http.Error(w, "method", http.StatusMethodNotAllowed)
}
})

return httptest.NewServer(mux)
}

Expand Down Expand Up @@ -197,8 +234,51 @@ var _ = Describe("httpapi.Client against the LocalAI admin REST surface", func()
Expect(bs[0].Installed).To(BeTrue())
})
})

Describe("Scheduling", func() {
It("lists scheduling configs", func() {
out, err := c.ListScheduling(ctx)
Expect(err).ToNot(HaveOccurred())
Expect(out).To(HaveLen(1))
Expect(out[0].ModelName).To(Equal("qwen"))
Expect(out[0].MinReplicas).To(Equal(1))
Expect(schedulingJSONKeys(&out[0])).ToNot(Or(
HaveKey("id"),
HaveKey("unsatisfiable_until"),
HaveKey("unsatisfiable_ticks"),
HaveKey("created_at"),
HaveKey("updated_at"),
))
})

It("gets one scheduling config", func() {
out, err := c.GetScheduling(ctx, "qwen")
Expect(err).ToNot(HaveOccurred())
Expect(out.ModelName).To(Equal("qwen"))
Expect(out.SpreadAll).To(BeTrue())
})

It("sets a scheduling config", func() {
out, err := c.SetScheduling(ctx, localaitools.SetSchedulingRequest{ModelName: "qwen", MinReplicas: 1, MaxReplicas: 2})
Expect(err).ToNot(HaveOccurred())
Expect(out.ModelName).To(Equal("qwen"))
Expect(out.MaxReplicas).To(Equal(2))
})

It("deletes a scheduling config", func() {
Expect(c.DeleteScheduling(ctx, "qwen")).To(Succeed())
})
})
})

func schedulingJSONKeys(config *localaitools.ModelSchedulingConfig) map[string]any {
var out map[string]any
b, err := json.Marshal(config)
Expect(err).ToNot(HaveOccurred())
Expect(json.Unmarshal(b, &out)).To(Succeed())
return out
}

var _ = Describe("Model aliases", func() {
Describe("ListAliases", func() {
It("passes the GET /api/aliases payload through unchanged", func() {
Expand Down
5 changes: 5 additions & 0 deletions pkg/mcp/localaitools/httpapi/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const (
routeBackendsKnown = "/backends/known"
routeBackendsApply = "/backends/apply"
routeNodes = "/api/nodes"
routeScheduling = "/api/nodes/scheduling"
routeVRAMEstimate = "/api/models/vram-estimate"
routeBranding = "/api/branding"
routeSettings = "/api/settings"
Expand Down Expand Up @@ -66,3 +67,7 @@ func routeVoiceProfileDelete(id string) string {
func routeNodeVRAMBudget(id string) string {
return "/api/nodes/" + url.PathEscape(id) + "/vram-budget"
}

func routeModelScheduling(modelName string) string {
return "/api/nodes/scheduling/" + url.PathEscape(modelName)
}
Loading