feat(vllm-cpp): move to vllm.cpp ABI v10 and expose jump-forward decoding

vllm.cpp landed ABI v10 on main while this branch was open. The backend's
runtime handshake refuses any library whose reported ABI differs from the
mirrors', so the pin and the Go PODs move together or not at all.

v10 appends one int32, `enable_jump_forward`, AFTER the v9 fields. Nothing else
in the config surface changed: the SGLang reconciliation that carried it
explicitly dropped its own duplicate scheduler_policy int in favour of the v9
`scheduling_policy` string this branch already wires, and a diff of EngineParams
and the server flags across the window turns up jump forward and nothing else.

So the exposure is one new knob, `engine_args.enable_jump_forward` - SGLang's
grammar-speed subset, which emits grammar-forced tokens without spending a model
step and therefore only affects constrained requests.

It is the SECOND tri-state on this struct, and it repeats the trap the first one
had: 0 is not "off", it is "defer" (to the environment here, to the model
capability for prefix caching), so an explicit `false` has to reach the engine as
2. The bool->tri-state helper and the log renderer are now shared rather than
duplicated per field, and named for the encoding instead of for prefix caching,
since the next tri-state will want them too. The docs say this outright, because
"omitting the key" and "setting it false" being different is not guessable.

The Go mirror grows the field plus an EXPLICIT trailing pad: the struct is
8-aligned and now ends on a lone int32, so it is 88 bytes rather than 84. The
offset assertions cover it, and the real-library handshake spec (VLLM_CPP_LIBRARY
against a CPU libvllm.so built at the new pin) confirms the version agrees:
43 specs pass, ABI reported 10.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude Code:claude-opus-5 [ClaudeCode]

Author: mudler

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=eec09bed5a03457837b499781c23d8e44f106813
+VLLM_CPP_VERSION?=f384edcd25950bdf785a17d9230f249b99bc841b
 
 # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the
 # server, examples and tests of the engine are never built here.

backend/go/vllm-cpp/backend.go

@@ -146,6 +146,7 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 		mp.MaxNumBatchedTokens = v.opts.maxNumBatchedTokens
 	}
 	mp.EnablePrefixCaching = v.opts.enablePrefixCaching
+	mp.EnableJumpForward = v.opts.enableJumpForward
 
 	// Every string below is borrowed by C for the duration of the load call
 	// only (the library copies what it keeps), so the backing slices just have
@@ -172,7 +173,8 @@ func (v *VllmCpp) Load(opts *pb.ModelOptions) error {
 		"blockSize", mp.BlockSize, "numBlocks", mp.NumBlocks,
 		"maxModelLen", mp.MaxModelLen, "maxNumSeqs", mp.MaxNumSeqs,
 		"maxNumBatchedTokens", mp.MaxNumBatchedTokens,
-		"prefixCaching", prefixCachingName(mp.EnablePrefixCaching),
+		"prefixCaching", triStateName(mp.EnablePrefixCaching),
+		"jumpForward", triStateName(mp.EnableJumpForward),
 		"schedulingPolicy", v.opts.schedulingPolicy,
 		"speculativeConfig", v.opts.speculativeConfig,
 		"kvTransferConfig", v.opts.kvTransferConfig)

backend/go/vllm-cpp/govllmcpp.go

@@ -1,6 +1,6 @@
 package main
 
-// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v9).
+// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v10).
 //
 // The structs below are hand-mirrored PODs of the C declarations, with
 // explicit padding so the Go layout matches the C layout on linux/darwin
@@ -18,24 +18,25 @@ import (
 )
 
 // abiVersion is the VLLM_ABI_VERSION this file mirrors (vllm.h).
-const abiVersion = 9
+const abiVersion = 10
 
-// Prefix-caching tri-state (vllm_model_params.enable_prefix_caching, ABI v7).
-// 0 is the model-capability default, which is what a config that says nothing
-// about prefix caching must produce.
+// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
+// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
+// "defer" - to the model capability for prefix caching, to the environment for
+// jump forward. Only 2 is an explicit off.
 const (
-	prefixCachingModelDefault int32 = 0
-	prefixCachingOn           int32 = 1
-	prefixCachingOff          int32 = 2
+	triStateDefer int32 = 0
+	triStateOn    int32 = 1
+	triStateOff   int32 = 2
 )
 
-// prefixCachingName renders the tri-state for the load log line, where "0"
-// would otherwise read as "off" rather than "whatever the model defaults to".
-func prefixCachingName(state int32) string {
+// triStateName renders a tri-state for the load log line, where "0" would
+// otherwise read as "off" rather than "whatever the default resolves to".
+func triStateName(state int32) string {
 	switch state {
-	case prefixCachingOn:
+	case triStateOn:
 		return "on"
-	case prefixCachingOff:
+	case triStateOff:
 		return "off"
 	default:
 		return "model-default"
@@ -47,9 +48,10 @@ const (
 	vllmOK = 0
 )
 
-// cModelParams mirrors vllm_model_params. Every field is naturally aligned on
-// LP64 (the int32 pairs sit together), so the Go layout needs no explicit
-// padding; the offsets are asserted in vllmcpp_test.go.
+// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the
+// interior needs no padding on LP64, but the struct is 8-aligned (it holds
+// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets
+// and total size are asserted in vllmcpp_test.go.
 type cModelParams struct {
 	ModelPath           uintptr // const char*
 	TokenizerConfigPath uintptr // const char*; NULL = <model_dir>/... (ABI v9)
@@ -64,6 +66,8 @@ type cModelParams struct {
 	MaxNumBatchedTokens int32   // <= 0 = per-arch default (ABI v9)
 	SchedulingPolicy    uintptr // const char*; NULL = "fcfs" (ABI v9)
 	KVTransferConfig    uintptr // const char* JSON; NULL = no connector (ABI v9)
+	EnableJumpForward   int32   // tri-state 0/1/2 (ABI v10)
+	_                   [4]byte // trailing pad to the struct's 8-byte alignment
 }
 
 // cSamplingParams mirrors vllm_sampling_params (ABI v2, structured fields

backend/go/vllm-cpp/options.go

@@ -42,6 +42,10 @@ type loadOptions struct {
 	// Automatic prefix caching tri-state (ABI v7): 0 = the model-capability
 	// default, 1 = force on, 2 = force off.
 	enablePrefixCaching int32
+	// Jump-forward decoding tri-state (ABI v10), SGLang's grammar-speed subset:
+	// 0 = defer to the environment (VT_ENABLE_JUMP_FORWARD, default off),
+	// 1 = force on, 2 = force off.
+	enableJumpForward int32
 	// Scheduler admission policy (ABI v9): "" = fcfs, else fcfs|priority|lpm.
 	schedulingPolicy string
 	// Engine-side parser selection (ABI v4/v5). Empty = the engine
@@ -100,7 +104,11 @@ func applyOptionsList(lo *loadOptions, options []string) {
 			lo.tokenizerConfigPath = strings.TrimSpace(v)
 		case "enable_prefix_caching", "enable_radix_attention":
 			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
-				lo.enablePrefixCaching = prefixCachingTriState(b)
+				lo.enablePrefixCaching = boolTriState(b)
+			}
+		case "enable_jump_forward":
+			if b, err := strconv.ParseBool(strings.TrimSpace(v)); err == nil {
+				lo.enableJumpForward = boolTriState(b)
 			}
 		}
 	}
@@ -145,23 +153,28 @@ func applyEngineArgs(lo *loadOptions, engineArgs string) {
 			lo.kvTransferConfig = jsonDocument(v, lo.kvTransferConfig, k)
 		case "enable_prefix_caching", "enable_radix_attention":
 			if b, ok := v.(bool); ok {
-				lo.enablePrefixCaching = prefixCachingTriState(b)
+				lo.enablePrefixCaching = boolTriState(b)
+			}
+		case "enable_jump_forward":
+			if b, ok := v.(bool); ok {
+				lo.enableJumpForward = boolTriState(b)
 			}
 		default:
 			xlog.Debug("[vllm-cpp] ignoring unknown engine_args key", "key", k)
 		}
 	}
 }
 
-// prefixCachingTriState maps a YAML/JSON boolean onto the C ABI's tri-state. An
+// boolTriState maps a YAML/JSON boolean onto the ABI's tri-state encoding. An
 // explicit `false` must reach the engine as force-OFF (2), NOT as the 0 that
-// means "let the model capability decide" - those differ for the hybrid archs,
-// which default the cache off, and for the dense ones, which default it on.
-func prefixCachingTriState(on bool) int32 {
+// means "defer". The difference is real in both directions: prefix caching
+// defaults ON for dense archs and OFF for hybrid ones, and jump forward defers
+// to VT_ENABLE_JUMP_FORWARD.
+func boolTriState(on bool) int32 {
 	if on {
-		return prefixCachingOn
+		return triStateOn
 	}
-	return prefixCachingOff
+	return triStateOff
 }
 
 // jsonDocument normalises an object-valued engine_args entry to a JSON string

backend/go/vllm-cpp/vllmcpp_test.go

@@ -35,7 +35,10 @@ var _ = Describe("C ABI struct mirrors", func() {
 		Expect(unsafe.Offsetof(p.MaxNumBatchedTokens)).To(Equal(uintptr(60)))
 		Expect(unsafe.Offsetof(p.SchedulingPolicy)).To(Equal(uintptr(64)))
 		Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72)))
-		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(80)))
+		Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(80)))
+		// 88, not 84: the struct is 8-aligned (it holds pointers), so the
+		// trailing int32 is padded out. Go pads identically.
+		Expect(unsafe.Sizeof(p)).To(Equal(uintptr(88)))
 	})
 
 	It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {
@@ -192,6 +195,23 @@ var _ = Describe("engine_args", func() {
 		Expect(lo.enablePrefixCaching).To(Equal(int32(1)))
 	})
 
+	It("maps enable_jump_forward onto its own tri-state", func() {
+		// ABI v10. Same tri-state shape as prefix caching, and the same trap:
+		// an explicit false must be force-OFF (2), not the 0 that defers to the
+		// environment.
+		on := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": true}`})
+		Expect(on.enableJumpForward).To(Equal(int32(1)))
+		off := parseOptions(&pb.ModelOptions{EngineArgs: `{"enable_jump_forward": false}`})
+		Expect(off.enableJumpForward).To(Equal(int32(2)))
+		unset := parseOptions(&pb.ModelOptions{EngineArgs: `{"max_num_seqs": 4}`})
+		Expect(unset.enableJumpForward).To(Equal(int32(0)))
+	})
+
+	It("reads enable_jump_forward from the legacy options list too", func() {
+		lo := parseOptions(&pb.ModelOptions{Options: []string{"enable_jump_forward:true"}})
+		Expect(lo.enableJumpForward).To(Equal(int32(1)))
+	})
+
 	It("lets engine_args override the legacy options list", func() {
 		lo := parseOptions(&pb.ModelOptions{
 			Options:    []string{"max_num_seqs:8", "block_size:16"},

docs/content/features/text-generation.md

@@ -948,6 +948,7 @@ engine_args:
 | `max_num_seqs` | Max concurrent sequences the scheduler admits | 8 |
 | `max_num_batched_tokens` | Per-step chunked-prefill token budget | per-arch (2048 dense, 4096/8192 MoE) |
 | `enable_prefix_caching` | Automatic prefix caching; `enable_radix_attention` is an accepted alias | model default |
+| `enable_jump_forward` | Jump-forward decoding, which emits grammar-forced tokens without a model step. Only affects constrained requests (`grammar`, JSON schema) | off |
 | `scheduling_policy` | `fcfs`, `priority`, or `lpm` | `fcfs` |
 | `tool_parser` / `reasoning_parser` | Force a parser instead of chat-template auto-detection | auto |
 | `tokenizer_config` | Override the `tokenizer_config.json` the chat template is read from | `<model_dir>/tokenizer_config.json` |
@@ -959,6 +960,12 @@ cost of decode latency for requests queued behind it. The default deliberately
 does not scale with `max_num_seqs`, which is what keeps a large concurrent
 prefill from blowing up the per-step activation on the hybrid architectures.
 
+`enable_prefix_caching` and `enable_jump_forward` are tri-state at the engine
+boundary: omitting the key defers to a default (the model's own capability for
+prefix caching, an environment variable for jump forward), while an explicit
+`false` forces the feature off. Those are genuinely different - prefix caching
+defaults *on* for dense models - so write the key only when you mean to override.
+
 #### Speculative decoding
 
 `speculative_config:` takes the same JSON object as vLLM's