docs(vllm): add server-side TTFT benchmark for the streaming tool-parser path

Self-contained stdlib-only script that measures time-to-first-token (TTFT)
for the vLLM backend's two streaming scenarios:

  - tool_call:  request mentions a tool; model is expected to call it
  - plain_text: request offers a tool but explicitly asks for prose

Use this to compare:
  - the buffer-all path (#10346)         → plain_text TTFT ≈ total response time
  - the native-streaming path (this PR)  → plain_text TTFT ≈ true first-token time

  python examples/vllm-bench/ttft_streaming_tool_parser.py \\
      --url http://localhost:8080 --model my-coder --runs 3

Lives under examples/ so it does not interfere with the test suite.

Author: Philipp Wacker

examples/vllm-bench/README.md

@@ -0,0 +1,44 @@
+# vLLM streaming + tool-parser benchmark
+
+A small, self-contained Python script (stdlib only) that measures
+time-to-first-token (TTFT) for the vLLM backend's streaming path with
+a tool parser configured.
+
+## Why this exists
+
+When a vLLM tool parser is active and a streaming chat completion is requested,
+LocalAI used to buffer the full generation to prevent raw tool-call markup
+(e.g. `<tool_call>...`) from leaking as `delta.content`. That was correct
+for tool-call responses, but it turned plain-text responses into effectively
+non-streaming — the client received nothing until the model finished.
+
+With native parser-side streaming (`parser.extract_tool_calls_streaming`,
+implemented by every concrete vLLM 0.23+ tool parser), each delta can be
+classified per-token: emit as content, emit as a structured tool_call, or
+suppress. This benchmark shows the difference.
+
+## Two scenarios
+
+| Scenario | Request | Expected outcome |
+|---|---|---|
+| `tool_call` | "What is the weather in Paris? Please use the tool." | Model calls `get_weather`. `delta.tool_calls` chunks; no content leak. |
+| `plain_text` | "Explain in 3 short sentences what a hash table is. Do NOT call any tool." | Model writes prose. With the streaming parser, content streams progressively; without it, the entire response arrives in one chunk. |
+
+## What the script reports
+
+For each scenario, across N runs:
+
+- `ttf_content_s` — time until the first `delta.content` chunk
+- `ttf_tool_s` — time until the first `delta.tool_calls` chunk
+- `n_content_chunks` — total number of distinct content deltas (1 = bundled, >1 = streamed)
+- `n_tool_chunks` — total tool_call deltas
+- `total_s` — total wall-clock until `[DONE]`
+- `finish_reason` — `tool_calls` / `stop` / `length`
+
+## Usage
+
+```bash
+python ttft_streaming_tool_parser.py --url http://localhost:8080 --model my-coder --runs 3
+```
+
+JSON results are written to `ttft_bench_<label>.json` (default label: `run`).

examples/vllm-bench/ttft_streaming_tool_parser.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""
+TTFT benchmark for the vLLM backend's streaming + tool-parser path.
+
+Measures time-to-first-token (TTFT) for two scenarios against a running
+LocalAI instance with a vLLM-backed chat model:
+
+  1. tool_call  — request mentions a tool; model is expected to call it
+  2. plain_text — request offers a tool but explicitly asks for prose
+
+Useful to validate the difference between:
+  - the buffer-all path  (#10346):       plain_text TTFT ≈ total response time
+  - the native-streaming path (this PR): plain_text TTFT ≈ true first-token time
+
+Usage:
+  python ttft_streaming_tool_parser.py \\
+      --url http://localhost:8080 --model my-coder --runs 3
+
+The script is self-contained (stdlib only — urllib, json, time, argparse).
+"""
+import argparse
+import json
+import sys
+import time
+import urllib.request
+
+DEFAULT_TOOLS = [{
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get current weather for a city",
+        "parameters": {
+            "type": "object",
+            "properties": {"city": {"type": "string"}},
+            "required": ["city"],
+        },
+    },
+}]
+
+SCENARIOS = [
+    {
+        "label": "tool_call",
+        "messages": [{"role": "user",
+                      "content": "What is the weather in Paris? Please use the tool."}],
+        "max_tokens": 80,
+    },
+    {
+        "label": "plain_text",
+        "messages": [{"role": "user",
+                      "content": "Explain in 3 short sentences what a hash table is. "
+                                 "Do NOT call any tool."}],
+        "max_tokens": 200,
+    },
+]
+
+
+def bench_one(url, model, messages, tools, max_tokens, timeout):
+    body = json.dumps({
+        "model": model,
+        "stream": True,
+        "tools": tools,
+        "messages": messages,
+        "max_tokens": max_tokens,
+    }).encode()
+    req = urllib.request.Request(
+        f"{url.rstrip('/')}/v1/chat/completions",
+        data=body, headers={"Content-Type": "application/json"},
+    )
+
+    t0 = time.perf_counter()
+    first_content = None
+    first_tool = None
+    n_content = 0
+    n_tool = 0
+    last = None
+    finish = None
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        for line in resp:
+            line = line.decode("utf-8", "replace").strip()
+            if not line.startswith("data: "):
+                continue
+            payload = line[6:]
+            if payload == "[DONE]":
+                break
+            try:
+                chunk = json.loads(payload)
+            except Exception:
+                continue
+            if not chunk.get("choices"):
+                continue
+            ch = chunk["choices"][0]
+            delta = ch.get("delta") or {}
+            now = time.perf_counter() - t0
+            if delta.get("content"):
+                if first_content is None:
+                    first_content = now
+                n_content += 1
+            if delta.get("tool_calls"):
+                if first_tool is None:
+                    first_tool = now
+                n_tool += 1
+            if ch.get("finish_reason"):
+                finish = ch["finish_reason"]
+            last = now
+    return {
+        "ttf_content_s": first_content,
+        "ttf_tool_s": first_tool,
+        "n_content_chunks": n_content,
+        "n_tool_chunks": n_tool,
+        "total_s": last,
+        "finish_reason": finish,
+    }
+
+
+def stats(values):
+    values = [v for v in values if v is not None]
+    if not values:
+        return "n/a"
+    return f"min={min(values):.3f}  avg={sum(values)/len(values):.3f}  max={max(values):.3f}"
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--url", default="http://localhost:8080",
+                   help="LocalAI base URL (default: %(default)s)")
+    p.add_argument("--model", default="coder", help="Model name (default: %(default)s)")
+    p.add_argument("--runs", type=int, default=3, help="Repetitions per scenario (default: %(default)s)")
+    p.add_argument("--timeout", type=int, default=120, help="Per-request timeout in seconds")
+    p.add_argument("--label", default="run",
+                   help="Tag for the JSON output file (default: %(default)s)")
+    args = p.parse_args()
+
+    print(f"=== TTFT Bench — {args.url}  model={args.model}  runs={args.runs} ===")
+    summary = {}
+    for sc in SCENARIOS:
+        print(f"\nScenario: {sc['label']}")
+        rows = []
+        for run in range(args.runs):
+            r = bench_one(args.url, args.model,
+                          sc["messages"], DEFAULT_TOOLS, sc["max_tokens"], args.timeout)
+            rows.append(r)
+            ttf_c = f"{r['ttf_content_s']:.3f}" if r["ttf_content_s"] is not None else "—"
+            ttf_t = f"{r['ttf_tool_s']:.3f}" if r["ttf_tool_s"] is not None else "—"
+            print(f"  run {run+1}/{args.runs}: "
+                  f"ttf_content={ttf_c}s  ttf_tool={ttf_t}s  "
+                  f"n_content={r['n_content_chunks']}  n_tool={r['n_tool_chunks']}  "
+                  f"total={r['total_s']:.2f}s  finish={r['finish_reason']}")
+        summary[sc["label"]] = rows
+
+    print("\n=== Summary (per scenario) ===")
+    for label, rows in summary.items():
+        print(f"[{label}]")
+        print(f"  ttf_content_s:    {stats(r['ttf_content_s'] for r in rows)}")
+        print(f"  ttf_tool_s:       {stats(r['ttf_tool_s']    for r in rows)}")
+        print(f"  n_content_chunks: {stats(r['n_content_chunks'] for r in rows)}")
+        print(f"  n_tool_chunks:    {stats(r['n_tool_chunks']    for r in rows)}")
+        print(f"  total_s:          {stats(r['total_s']        for r in rows)}")
+
+    out = f"ttft_bench_{args.label}.json"
+    with open(out, "w") as f:
+        json.dump(summary, f, indent=2)
+    print(f"\nSaved to {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())