Merge branch 'master' into feat/vllm-cpp-engine-args

Author: localai-org-maint-bot

.github/backend-matrix.yml

@@ -66,6 +66,34 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-kokoro'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'true'
+    backend: "kokoro"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-kokoro'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'true'
+    backend: "kokoro"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""

backend/backend.proto

@@ -508,6 +508,12 @@ message ProxyOptions {
   string api_key_file = 5;
   string upstream_model = 6;
   int32 request_timeout_seconds = 7;
+  // cache_prompt enables automatic Anthropic prompt-cache breakpoints
+  // (cache_control: ephemeral) on the stable prefix — system, tools, and
+  // the last message block — when translating to the Anthropic provider.
+  // Cuts input cost on repeated/agentic calls (cache read = 0.1x). Only
+  // meaningful for mode=translate + provider=anthropic; ignored otherwise.
+  bool cache_prompt = 8;
 }
 
 message Result {

backend/go/cloud-proxy/provider_anthropic.go

@@ -32,7 +32,9 @@ import (
 type anthropicRequest struct {
 	Model         string               `json:"model"`
 	MaxTokens     int32                `json:"max_tokens"`
-	System        string               `json:"system,omitempty"`
+	// System is `any`: a bare string normally, or []anthropicSystemBlock
+	// when cache_prompt is on (the block form carries cache_control).
+	System        any                  `json:"system,omitempty"`
 	Messages      []anthropicMessage   `json:"messages"`
 	Stream        bool                 `json:"stream,omitempty"`
 	Temperature   *float64             `json:"temperature,omitempty"`
@@ -52,9 +54,30 @@ type anthropicMessage struct {
 }
 
 type anthropicTool struct {
-	Name        string          `json:"name"`
-	Description string          `json:"description,omitempty"`
-	InputSchema json.RawMessage `json:"input_schema"`
+	Name         string                 `json:"name"`
+	Description  string                 `json:"description,omitempty"`
+	InputSchema  json.RawMessage        `json:"input_schema"`
+	CacheControl *anthropicCacheControl `json:"cache_control,omitempty"`
+}
+
+// anthropicCacheControl marks a prompt-cache breakpoint. Anthropic caches
+// everything up to and including a block tagged {"type":"ephemeral"} (5-min
+// TTL) and serves that prefix at the cache-read rate (0.1x input) on later
+// calls that share it — the win on agentic/multi-turn workloads.
+type anthropicCacheControl struct {
+	Type string `json:"type"` // "ephemeral"
+}
+
+// ephemeralCacheControl is the single reused breakpoint marker.
+var ephemeralCacheControl = &anthropicCacheControl{Type: "ephemeral"}
+
+// anthropicSystemBlock is the block form of the top-level system field.
+// Anthropic accepts system as a bare string OR a list of text blocks; the
+// block form is required to attach cache_control to the system prompt.
+type anthropicSystemBlock struct {
+	Type         string                 `json:"type"` // "text"
+	Text         string                 `json:"text"`
+	CacheControl *anthropicCacheControl `json:"cache_control,omitempty"`
 }
 
 // anthropicToolChoice mirrors the four shapes Anthropic accepts:
@@ -81,8 +104,9 @@ type anthropicContentBlock struct {
 	// Tool-result block fields. tool_result uses `content` (not
 	// `text`) and pairs with `tool_use_id`; modelling them as
 	// distinct fields avoids ambiguity at marshal time.
-	ToolUseID     string `json:"tool_use_id,omitempty"`
-	ResultContent string `json:"content,omitempty"`
+	ToolUseID     string                 `json:"tool_use_id,omitempty"`
+	ResultContent string                 `json:"content,omitempty"`
+	CacheControl  *anthropicCacheControl `json:"cache_control,omitempty"`
 }
 
 type anthropicResponse struct {
@@ -156,6 +180,11 @@ func buildAnthropicRequest(opts *pb.PredictOptions, cfg *proxyConfig, stream boo
 	if req.ToolChoice != nil && req.ToolChoice.Type == anthropicToolChoiceNone {
 		req.Tools, req.ToolChoice = nil, nil
 	}
+	// Prompt-cache breakpoint on the last tool: Anthropic caches the entire
+	// tool block up to the marked tool — usually a large, fully stable prefix.
+	if cfg.cachePrompt && len(req.Tools) > 0 {
+		req.Tools[len(req.Tools)-1].CacheControl = ephemeralCacheControl
+	}
 
 	var systemParts []string
 	for _, m := range opts.GetMessages() {
@@ -189,15 +218,54 @@ func buildAnthropicRequest(opts *pb.PredictOptions, cfg *proxyConfig, stream boo
 			})
 		}
 	}
-	req.System = strings.Join(systemParts, "\n\n")
+	// System: block form (with cache_control) when caching is on, else the
+	// bare string. Only set when non-empty so `omitempty` still drops it.
+	if len(systemParts) > 0 {
+		joined := strings.Join(systemParts, "\n\n")
+		if cfg.cachePrompt {
+			req.System = []anthropicSystemBlock{{Type: "text", Text: joined, CacheControl: ephemeralCacheControl}}
+		} else {
+			req.System = joined
+		}
+	}
 
 	if len(req.Messages) == 0 && opts.GetPrompt() != "" {
 		req.Messages = []anthropicMessage{{Role: "user", Content: opts.GetPrompt()}}
 	}
 
+	// Prompt-cache breakpoint on the final message block caches the whole
+	// conversation prefix up to the newest turn. With the system + tools
+	// breakpoints above, Anthropic serves the entire stable head at the
+	// cache-read rate on the next agentic iteration (max 4 breakpoints; we
+	// use at most 3, so we never exceed the limit).
+	if cfg.cachePrompt {
+		markLastMessageCacheable(req.Messages)
+	}
+
 	return json.Marshal(req)
 }
 
+// markLastMessageCacheable tags the final block of the last message with a
+// cache_control breakpoint. String content is promoted to a single text
+// block so the marker has somewhere to attach; block content gets the marker
+// on its last element.
+func markLastMessageCacheable(msgs []anthropicMessage) {
+	if len(msgs) == 0 {
+		return
+	}
+	last := &msgs[len(msgs)-1]
+	switch c := last.Content.(type) {
+	case string:
+		if c != "" {
+			last.Content = []anthropicContentBlock{{Type: "text", Text: c, CacheControl: ephemeralCacheControl}}
+		}
+	case []anthropicContentBlock:
+		if len(c) > 0 {
+			c[len(c)-1].CacheControl = ephemeralCacheControl
+		}
+	}
+}
+
 // appendToolResult appends a tool_result block as a user message,
 // merging into a preceding user message that already carries blocks.
 // Anthropic concatenates consecutive same-role messages on its end,

backend/go/cloud-proxy/provider_anthropic_test.go

@@ -328,3 +328,62 @@ func TestBuildAnthropic_RoundTripsAssistantToolCalls(t *testing.T) {
 	g.Expect(r0["tool_use_id"]).To(Equal("call_abc"))
 	g.Expect(r0["content"]).To(Equal(`{"models":["a","b"]}`))
 }
+
+// TestPredict_Anthropic_PromptCache verifies that cache_prompt injects
+// exactly the intended cache_control breakpoints (system, last tool, last
+// message) when on, and none when off — asserting on the raw upstream body
+// because System becomes a block list that the typed struct hides.
+func TestPredict_Anthropic_PromptCache(t *testing.T) {
+	g := NewWithT(t)
+
+	// run issues one translate Predict and returns the raw body the fake
+	// Anthropic upstream received.
+	run := func(cachePrompt bool) string {
+		var rawBody string
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			b, _ := io.ReadAll(r.Body)
+			rawBody = string(b)
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = io.WriteString(w, `{"id":"m","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"model":"claude-3-5-sonnet-20241022","usage":{"input_tokens":5,"output_tokens":2}}`)
+		}))
+		defer srv.Close()
+
+		t.Setenv("CLOUD_PROXY_ANTHROPIC_FAKE", "sk-ant-fake")
+		cp := NewCloudProxy()
+		err := cp.Load(&pb.ModelOptions{
+			Model: "claude-local",
+			Proxy: &pb.ProxyOptions{
+				UpstreamUrl:   srv.URL,
+				Mode:          modeTranslate,
+				Provider:      providerAnthropic,
+				ApiKeyEnv:     "CLOUD_PROXY_ANTHROPIC_FAKE",
+				UpstreamModel: "claude-3-5-sonnet-20241022",
+				CachePrompt:   cachePrompt,
+			},
+		})
+		g.Expect(err).NotTo(HaveOccurred())
+
+		_, err = cp.Predict(&pb.PredictOptions{
+			Messages: []*pb.Message{
+				{Role: "system", Content: "be brief"},
+				{Role: "user", Content: "hello"},
+			},
+			Tools:  `[{"type":"function","function":{"name":"t","parameters":{"type":"object"}}}]`,
+			Tokens: 32,
+		})
+		g.Expect(err).NotTo(HaveOccurred())
+		return rawBody
+	}
+
+	// cache_prompt ON: three ephemeral breakpoints (system + last tool +
+	// last message), and system is emitted in block form.
+	on := run(true)
+	g.Expect(strings.Count(on, `"cache_control":{"type":"ephemeral"}`)).To(Equal(3),
+		"expected 3 breakpoints (system, tool, last message); body=%s", on)
+	g.Expect(on).To(ContainSubstring(`"system":[{"type":"text","text":"be brief"`))
+
+	// cache_prompt OFF: no breakpoints, system stays a bare string.
+	off := run(false)
+	g.Expect(off).NotTo(ContainSubstring("cache_control"))
+	g.Expect(off).To(ContainSubstring(`"system":"be brief"`))
+}

backend/go/cloud-proxy/proxy.go

@@ -48,6 +48,7 @@ type proxyConfig struct {
 	upstreamModel string
 	localModel    string // ModelOptions.Model — fallback when upstream_model is unset
 	apiKey        string // resolved at Load time
+	cachePrompt   bool   // inject Anthropic prompt-cache breakpoints (translate+anthropic)
 }
 
 func NewCloudProxy() *CloudProxy {
@@ -106,6 +107,7 @@ func (c *CloudProxy) Load(opts *pb.ModelOptions) error {
 		upstreamModel: po.GetUpstreamModel(),
 		localModel:    opts.GetModel(),
 		apiKey:        key,
+		cachePrompt:   po.GetCachePrompt(),
 	})
 	xlog.Info("cloud-proxy: ready",
 		"upstream", po.GetUpstreamUrl(),

backend/index.yaml

@@ -1455,6 +1455,7 @@
   alias: "kokoro"
   name: "kokoro"
   capabilities:
+    default: "cpu-kokoro"
     nvidia: "cuda12-kokoro"
     intel: "intel-kokoro"
     amd: "rocm-kokoro"
@@ -5186,11 +5187,22 @@
 - !!merge <<: *kokoro
   name: "kokoro-development"
   capabilities:
+    default: "cpu-kokoro-development"
     nvidia: "cuda12-kokoro-development"
     intel: "intel-kokoro-development"
     amd: "rocm-kokoro-development"
     nvidia-l4t: "nvidia-l4t-kokoro-development"
     metal: "metal-kokoro-development"
+- !!merge <<: *kokoro
+  name: "cpu-kokoro"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-kokoro"
+  mirrors:
+    - localai/localai-backends:latest-cpu-kokoro
+- !!merge <<: *kokoro
+  name: "cpu-kokoro-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-cpu-kokoro"
+  mirrors:
+    - localai/localai-backends:master-cpu-kokoro
 - !!merge <<: *kokoro
   name: "cuda12-kokoro-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-kokoro"

backend/python/mlx-vlm/requirements-mps.txt

@@ -1 +1,3 @@
-git+https://github.com/Blaizzy/mlx-vlm@v0.4.4
+git+https://github.com/Blaizzy/mlx-vlm@v0.4.4
+torch
+torchvision

core/backend/options.go

@@ -475,6 +475,7 @@ func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
 			ApiKeyFile:            c.Proxy.APIKeyFile,
 			UpstreamModel:         c.Proxy.UpstreamModel,
 			RequestTimeoutSeconds: int32(c.Proxy.RequestTimeoutSeconds),
+			CachePrompt:           c.Proxy.CachePrompt,
 		}
 	}
 

core/config/meta/registry.go

@@ -822,6 +822,13 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Min:         f64(0),
 			Order:       213,
 		},
+		"proxy.cache_prompt": {
+			Section:     "proxy",
+			Label:       "Proxy Anthropic Prompt Cache",
+			Description: "Inject Anthropic prompt-cache breakpoints (cache_control: ephemeral) on the stable prefix (system, tools, last message) when mode is translate and provider is anthropic. Serves the repeated prefix at the cache-read rate on multi-turn/agentic calls. No effect otherwise.",
+			Component:   "checkbox",
+			Order:       214,
+		},
 
 		// --- MITM intercept hosts ---
 		// Each host listed here is claimed by this model config; the

core/config/model_config.go

@@ -232,6 +232,15 @@ type ProxyConfig struct {
 	// means no per-request timeout (only the request context, which
 	// is bound to the client connection, applies).
 	RequestTimeoutSeconds int `yaml:"request_timeout_seconds,omitempty" json:"request_timeout_seconds,omitempty"`
+
+	// CachePrompt enables automatic Anthropic prompt-cache breakpoints
+	// (cache_control: ephemeral) on the stable prefix — system prompt,
+	// tools, and the last message block — when mode=translate and
+	// provider=anthropic. Anthropic then serves the repeated prefix at
+	// the cache-read rate (0.1x input), which sharply cuts cost on
+	// agentic/multi-turn workloads that re-send a large stable prefix.
+	// No effect for passthrough mode or non-Anthropic providers.
+	CachePrompt bool `yaml:"cache_prompt,omitempty" json:"cache_prompt,omitempty"`
 }
 
 // Proxy mode names. Validate() normalises an empty Mode to