feat(vllm-cpp): wire the full engine config surface through engine_args

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=9e1c9025ae61167a3335454d7cc0de6093c21845
+VLLM_CPP_VERSION?=6d4fb5e7ae13eb8a30fdd3f5e8df8a3d8a83143b
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.
@@ -56,6 +56,11 @@ endif
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
 	LIB=libvllm.dylib
+	# Apple Clang diagnoses a pair of constant-folded array bounds in the Metal
+	# build as a GNU extension. Disable that diagnostic because vllm.cpp appends
+	# target-local -Werror after these global flags, overriding -Wno-error.
+	CMAKE_ARGS+=-DCMAKE_CXX_FLAGS=-Wno-gnu-folding-constant
+	CMAKE_ARGS+=-DCMAKE_OBJCXX_FLAGS=-Wno-gnu-folding-constant
 else
 	LIB=libvllm.so
 endif

backend/go/vllm-cpp/backend.go

@@ -109,41 +109,79 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 
 	v.opts = parseOptions(opts)
 
+	// A DFlash draft is a second checkpoint the engine opens by path, and the
+	// engine never downloads one. Resolve it against LocalAI's models directory
+	// now so a repo-id spelling works, and so a missing draft fails here with an
+	// actionable message rather than as an HF-cache miss inside the load.
+	resolvedSpec, err := resolveDraftModelPath(v.opts.speculativeConfig, opts.ModelPath)
+	if err != nil {
+		return err
+	}
+	v.opts.speculativeConfig = resolvedSpec
+
 	mp := defaultModelParams()
 	if v.opts.blockSize > 0 {
 		mp.BlockSize = v.opts.blockSize
 	}
 	if v.opts.numBlocks > 0 {
 		mp.NumBlocks = v.opts.numBlocks
 	}
+	// Sequence-length precedence, narrowest source last: context_size is the
+	// generic LocalAI knob every backend honours, max_model_len is the
+	// vLLM-specific one, and engine_args.max_model_len is the explicit
+	// vllm-cpp override.
 	if opts.ContextSize > 0 {
 		mp.MaxModelLen = opts.ContextSize
 	}
+	if opts.MaxModelLen > 0 {
+		mp.MaxModelLen = opts.MaxModelLen
+	}
+	if v.opts.maxModelLen > 0 {
+		mp.MaxModelLen = v.opts.maxModelLen
+	}
 	if v.opts.maxNumSeqs > 0 {
 		mp.MaxNumSeqs = v.opts.maxNumSeqs
 	}
+	if v.opts.maxNumBatchedTokens > 0 {
+		mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens
+	}
+	mp.EnablePrefixCaching = v.opts.enablePrefixCaching
+	mp.EnableJumpForward = v.opts.enableJumpForward
 
+	// Every string below is borrowed by C for the duration of the load call
+	// only (the library copies what it keeps), so the backing slices just have
+	// to outlive vllmEngineLoad - hence the single KeepAlive after it.
 	modelC := cString(model)
 	mp.ModelPath = uintptr(unsafe.Pointer(&modelC[0])) // #nosec G103 -- borrowed by C for the load call only
-	var toolParserC, reasoningParserC []byte
-	if v.opts.toolParser != "" {
-		toolParserC = cString(v.opts.toolParser)
-		mp.ToolParser = uintptr(unsafe.Pointer(&toolParserC[0])) // #nosec G103 -- borrowed by C for the load call only
-	}
-	if v.opts.reasoningParser != "" {
-		reasoningParserC = cString(v.opts.reasoningParser)
-		mp.ReasoningParser = uintptr(unsafe.Pointer(&reasoningParserC[0])) // #nosec G103 -- borrowed by C for the load call only
+	keep := [][]byte{modelC}
+	setStr := func(dst *uintptr, s string) {
+		if s == "" {
+			return
+		}
+		b := cString(s)
+		keep = append(keep, b)
+		*dst = uintptr(unsafe.Pointer(&b[0])) // #nosec G103 -- borrowed by C for the load call only
 	}
+	setStr(&mp.ToolParser, v.opts.toolParser)
+	setStr(&mp.ReasoningParser, v.opts.reasoningParser)
+	setStr(&mp.SpeculativeConfig, v.opts.speculativeConfig)
+	setStr(&mp.KVTransferConfig, v.opts.kvTransferConfig)
+	setStr(&mp.SchedulingPolicy, v.opts.schedulingPolicy)
+	setStr(&mp.TokenizerConfigPath, v.opts.tokenizerConfigPath)
 
 	xlog.Info("[vllm-cpp] Load", "model", model, "engine", vllmVersion(),
 		"blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks,
-		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs)
+		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs,
+		"maxNumBatchedTokens", mp.MaxNumBatchedTokens,
+		"prefixCaching", triStateName(mp.EnablePrefixCaching),
+		"jumpForward", triStateName(mp.EnableJumpForward),
+		"schedulingPolicy", v.opts.schedulingPolicy,
+		"speculativeConfig", v.opts.speculativeConfig,
+		"kvTransferConfig", v.opts.kvTransferConfig)
 
 	var engine uintptr
 	rc := vllmEngineLoad(unsafe.Pointer(&mp), unsafe.Pointer(&engine)) // #nosec G103 -- POD out-params
-	runtime.KeepAlive(modelC)
-	runtime.KeepAlive(toolParserC)
-	runtime.KeepAlive(reasoningParserC)
+	runtime.KeepAlive(keep)
 	if rc != vllmOK {
 		return fmt.Errorf("vllm-cpp: engine load failed: %s", vllmLastError())
 	}

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v2).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -18,23 +18,56 @@ import (
 )
 
 // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 5
+const abiVersion = 10
+
+// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
+// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
+// "defer" - to the model capability for prefix caching, to the environment for
+// jump forward. Only 2 is an explicit off.
+const (
+	triStateDefer int32 = 0
+	triStateOn    int32 = 1
+	triStateOff   int32 = 2
+)
+
+// triStateName renders a tri-state for the load log line, where "0" would
+// otherwise read as "off" rather than "whatever the default resolves to".
+func triStateName(state int32) string {
+	switch state {
+	case triStateOn:
+		return "on"
+	case triStateOff:
+		return "off"
+	default:
+		return "model-default"
+	}
+}
 
 // vllm_status (vllm.h).
 const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params.
+// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the
+// interior needs no padding on LP64, but the struct is 8-aligned (it holds
+// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets
+// and total size are asserted in vllmcpp_test.go.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
-	TokenizerConfigPath uintptr // const char*
+	TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
 	BlockSize           int32
 	NumBlocks           int32
 	MaxModelLen         int32
 	MaxNumSeqs          int32
 	ToolParser          uintptr // const char*; NULL = auto-detect (ABI v4)
 	ReasoningParser     uintptr // const char*; NULL = auto-detect (ABI v5)
+	SpeculativeConfig   uintptr // const char* JSON; NULL = no speculation (ABI v6)
+	EnablePrefixCaching int32   // tri-state 0/1/2 (ABI v7)
+	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
+	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
+	KVTransferConfig    uintptr // const char* JSON; NULL = no connector (ABI v9)
+	EnableJumpForward   int32   // tri-state 0/1/2 (ABI v10)
+	_                   [4]byte // trailing pad to the struct's 8-byte alignment
 }
 
 // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields
@@ -65,6 +98,12 @@ type cSamplingParams struct {
 	StructuredGrammar    uintptr // const char*
 	StructuredJSONObject int32
 	_                    [4]byte
+	// ABI v8 tail. LocalAI installs no custom logits processor, but the fields
+	// MUST be mirrored: the C side reads them off the pointer we hand it, so a
+	// Go struct that stopped at StructuredJSONObject would have the engine read
+	// 16 bytes past our allocation and call whatever garbage sat there.
+	LogitsProcessor         uintptr // vllm_logits_processor; NULL = none
+	LogitsProcessorUserData uintptr // void*
 }
 
 // cCompletion mirrors vllm_completion.