feat(vbgui-fg): tokenizer playground — encode_visualize + side-by-side panel

Stage F-G of the Visual Builder GUI epic (cppmega-mlx-o0k). Backend
endpoints + React panel that load any HF-style tokenizer.json and
emit per-token byte spans, with up to 3 panels side-by-side for A/B/C
comparison.

Backend (cppmega_v4.jsonrpc.tokenizer_methods):
- tokenizer.encode_visualize — loads tokenizer.json via HF tokenizers
  Python lib, encodes input text, emits TokenSpan list (id / literal /
  start / end / is_special) plus bytes-per-token avg + max and the
  tokenizer's capability snapshot (vocab size, special-id contract,
  decoder kind). Per-source LRU cache.
- tokenizer.list_presets — returns the 12-entry preset library
  (cppmega_v3 / nanochat_v3 / gpt-4-o200k / gpt-3.5-cl100k / llama-3
  / mistral / gemma / qwen / deepseek-v3 / phi-3 / claude / gpt-2-p50k).
- Dispatcher: both methods wired; METHOD_REGISTRY locks the names.

Frontend (vbgui/src/components/TokenizerPlayground.tsx):
- Single textarea drives N panels (cap 3 via maxPanels prop).
- Each panel: source input + Encode button + metrics line +
  color-chip grid keyed on token id (specials get a red chip).
- Cross-panel hover highlight: hover a chip in one panel → every chip
  in every panel whose byte span overlaps gets a blue ring (cross-
  panel highlight bus per VBPlan §9.5.3, no global state — props).
- Error envelope from RpcClient surfaces inline per panel.

Tests:
- Python (11): encode round-trip, special detection (BOS in vendored
  tokenizer), capability surface, cache hit, sub-50ms warm latency,
  missing-source error, list_presets shape, dispatcher round-trip
  for both endpoints, METHOD_REGISTRY lock.
- TypeScript (6): renders input + add button, initialSources →
  panels, maxPanels cap, Encode populates chips + metrics, Remove
  drops panel, error envelope surfaces inline.

Full v4 regression: 2079 passed / 5 skipped / 15 xfailed / 0 failed.
Vbgui: 71 tests passed (65 + 6 new).

Closes cppmega-mlx-o0k.7.

Author: apstenku123

cppmega_v4/jsonrpc/dispatcher.py

@@ -22,6 +22,11 @@
     suggest_sharding,
     verify,
 )
+from cppmega_v4.jsonrpc.tokenizer_methods import (
+    EncodeVisualizeParams,
+    encode_visualize,
+    list_presets as tokenizer_list_presets,
+)
 from cppmega_v4.jsonrpc.schema import (
     BuildPresetSpecsParams,
     ErrorCode,
@@ -67,6 +72,10 @@
         PipelineRunParams,
         lambda p, c: _pipeline_run(p),
     ),
+    "tokenizer.encode_visualize": (
+        EncodeVisualizeParams,
+        lambda p, c: encode_visualize(p, cache=c),
+    ),
 }
 
 
@@ -115,6 +124,12 @@ def dispatch(
     if request.method == "backend.status":
         return JsonRpcResponse(id=request.id, result={"status": "ok"})
 
+    if request.method == "tokenizer.list_presets":
+        return JsonRpcResponse(
+            id=request.id,
+            result=tokenizer_list_presets().model_dump(mode="json"),
+        )
+
     route = _ROUTES.get(request.method)
     if route is None:
         return _error_response(

cppmega_v4/jsonrpc/schema.py

@@ -532,4 +532,6 @@ class PipelineRunResult(BaseModel):
     "probe.run",
     "pipeline.run",
     "backend.status",
+    "tokenizer.encode_visualize",
+    "tokenizer.list_presets",
 })

cppmega_v4/jsonrpc/tokenizer_methods.py

@@ -0,0 +1,186 @@
+"""Backend tokenizer playground — F-G handlers.
+
+Implements ``tokenizer.encode_visualize`` and ``tokenizer.list_presets``
+JSON-RPC methods. Backed by the HuggingFace ``tokenizers`` Python
+library so any ``tokenizer.json`` source works uniformly. Tiktoken /
+SentencePiece formats are folded in later via the runtime registry on
+the frontend (lazy-loaded WASM); the backend exposes one shape.
+
+The encode_visualize result carries per-token byte spans + decoded
+literals so the GUI can paint char-aligned color chips and emit
+hover-cross-panel highlight events.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+from tokenizers import Tokenizer
+
+from cppmega_v4.jsonrpc.cache import LRUCache
+from cppmega_v4.probe import TokenizerCapabilities, introspect_tokenizer
+
+
+# Built-in preset library — paths are placeholders that map to real
+# tokenizer.json files when shipped via the widget bundle. Resolving
+# real paths is the GUI's job; the backend just declares the names.
+PRESET_LIBRARY: tuple[str, ...] = (
+    "cppmega_v3", "nanochat_v3",
+    "gpt-4-o200k", "gpt-3.5-cl100k", "gpt-2-p50k",
+    "llama-3", "mistral", "gemma", "qwen",
+    "deepseek-v3", "phi-3", "claude",
+)
+
+
+# ---------------------------------------------------------------------------
+# Schemas
+# ---------------------------------------------------------------------------
+
+
+class TokenSpan(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: int
+    text: str
+    start: int
+    end: int
+    is_special: bool = False
+
+
+class EncodeVisualizeParams(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    tokenizer_source: str
+    text: str
+    add_special_tokens: bool = True
+
+
+class EncodeVisualizeResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    tokens: list[TokenSpan]
+    token_count: int
+    bytes_total: int
+    bytes_per_token_avg: float
+    bytes_per_token_max: int
+    capabilities: dict[str, Any]
+    elapsed_ms: float
+
+
+class ListPresetsResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    presets: list[str] = Field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer cache — separate from the main RPC cache because keys differ.
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class _LoadedTokenizer:
+    tokenizer: Tokenizer
+    capabilities: TokenizerCapabilities
+
+
+_TOKENIZER_CACHE: dict[str, _LoadedTokenizer] = {}
+
+
+def _load(source: str) -> _LoadedTokenizer:
+    hit = _TOKENIZER_CACHE.get(source)
+    if hit is not None:
+        return hit
+    path = Path(source)
+    if not path.is_file():
+        raise FileNotFoundError(f"tokenizer source not found: {source}")
+    tok = Tokenizer.from_file(str(path))
+    caps = introspect_tokenizer(path)
+    loaded = _LoadedTokenizer(tokenizer=tok, capabilities=caps)
+    _TOKENIZER_CACHE[source] = loaded
+    return loaded
+
+
+# ---------------------------------------------------------------------------
+# Handlers
+# ---------------------------------------------------------------------------
+
+
+def encode_visualize(
+    params: EncodeVisualizeParams,
+    *,
+    cache: LRUCache | None = None,
+) -> EncodeVisualizeResult:
+    """Encode ``text`` through ``tokenizer_source`` and emit per-token spans."""
+    cache_key = _cache_key(params)
+    if cache is not None:
+        hit = cache.get(cache_key)
+        if hit is not None:
+            return hit
+
+    t0 = time.perf_counter()
+    loaded = _load(params.tokenizer_source)
+    encoded = loaded.tokenizer.encode(
+        params.text, add_special_tokens=params.add_special_tokens,
+    )
+
+    text_bytes = params.text.encode("utf-8")
+    special_ids = set(loaded.capabilities.special_ids.values())
+
+    spans: list[TokenSpan] = []
+    max_bytes = 0
+    for tok_id, tok_text, (start, end) in zip(
+        encoded.ids, encoded.tokens, encoded.offsets,
+    ):
+        span_bytes = max(0, end - start)
+        max_bytes = max(max_bytes, span_bytes)
+        spans.append(TokenSpan(
+            id=int(tok_id),
+            text=tok_text,
+            start=int(start),
+            end=int(end),
+            is_special=int(tok_id) in special_ids,
+        ))
+
+    avg = (sum((s.end - s.start) for s in spans) / len(spans)) if spans else 0.0
+    elapsed = (time.perf_counter() - t0) * 1000.0
+    out = EncodeVisualizeResult(
+        tokens=spans,
+        token_count=len(spans),
+        bytes_total=len(text_bytes),
+        bytes_per_token_avg=round(avg, 3),
+        bytes_per_token_max=max_bytes,
+        capabilities=_caps_to_dict(loaded.capabilities),
+        elapsed_ms=elapsed,
+    )
+    if cache is not None:
+        cache.set(cache_key, out)
+    return out
+
+
+def list_presets() -> ListPresetsResult:
+    return ListPresetsResult(presets=list(PRESET_LIBRARY))
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _cache_key(p: EncodeVisualizeParams) -> str:
+    # No layout fields here, so canonical JSON via sorted-keys is fine.
+    return f"tokenize::{p.tokenizer_source}::{p.add_special_tokens}::{hash(p.text)}"
+
+
+def _caps_to_dict(c: TokenizerCapabilities) -> dict[str, Any]:
+    return {
+        "vocab_size": c.vocab_size,
+        "special_ids": dict(c.special_ids),
+        "has_fim": c.has_fim,
+        "has_space_nl": c.has_space_nl,
+        "has_code_start": c.has_code_start,
+        "has_instruction": c.has_instruction,
+        "byte_roundtrip": c.byte_roundtrip,
+        "decoder_kind": c.decoder_kind,
+        "source": c.source,
+    }

tests/v4/test_tokenizer_playground.py

@@ -0,0 +1,127 @@
+"""F-G tokenizer playground tests — encode_visualize + list_presets."""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+import pytest
+
+from cppmega_v4.jsonrpc import LRUCache, dispatch
+from cppmega_v4.jsonrpc.tokenizer_methods import (
+    EncodeVisualizeParams,
+    PRESET_LIBRARY,
+    encode_visualize,
+    list_presets,
+)
+
+
+_VENDORED = "cppmega_mlx/tokenizer/tokenizer.json"
+
+
+# ---------------------------------------------------------------------------
+# encode_visualize
+# ---------------------------------------------------------------------------
+
+
+def test_encode_visualize_returns_spans():
+    p = EncodeVisualizeParams(tokenizer_source=_VENDORED, text="hello world")
+    r = encode_visualize(p)
+    assert r.token_count > 0
+    assert all(s.start >= 0 and s.end >= s.start for s in r.tokens)
+    assert r.bytes_total == len("hello world".encode("utf-8"))
+
+
+def test_encode_visualize_marks_specials_when_present():
+    p = EncodeVisualizeParams(tokenizer_source=_VENDORED, text="<BOS>code")
+    r = encode_visualize(p, cache=LRUCache())
+    # At least one token id should be in the special-id set (BOS = 2).
+    assert any(s.is_special for s in r.tokens)
+
+
+def test_encode_visualize_capabilities_match_vendored_tokenizer():
+    p = EncodeVisualizeParams(tokenizer_source=_VENDORED, text="x")
+    r = encode_visualize(p)
+    caps = r.capabilities
+    assert caps["vocab_size"] > 0
+    assert caps["has_fim"] is True
+    assert caps["has_space_nl"] is True
+
+
+def test_encode_visualize_caches_repeat_calls():
+    cache = LRUCache(capacity=4)
+    p = EncodeVisualizeParams(tokenizer_source=_VENDORED, text="abc")
+    encode_visualize(p, cache=cache)
+    encode_visualize(p, cache=cache)
+    assert cache.stats()["hits"] >= 1
+
+
+def test_encode_visualize_under_50ms_short_text():
+    p = EncodeVisualizeParams(tokenizer_source=_VENDORED, text="short input")
+    # Warm tokenizer cache
+    encode_visualize(p)
+    t0 = time.perf_counter()
+    encode_visualize(p)
+    elapsed_ms = (time.perf_counter() - t0) * 1000.0
+    assert elapsed_ms < 50, f"warm encode took {elapsed_ms:.1f} ms"
+
+
+def test_encode_visualize_missing_source_raises():
+    p = EncodeVisualizeParams(tokenizer_source="/nonexistent/tok.json", text="x")
+    with pytest.raises(FileNotFoundError):
+        encode_visualize(p)
+
+
+# ---------------------------------------------------------------------------
+# list_presets
+# ---------------------------------------------------------------------------
+
+
+def test_list_presets_returns_known_library():
+    r = list_presets()
+    assert len(r.presets) == len(PRESET_LIBRARY)
+    assert "gpt-4-o200k" in r.presets
+    assert "cppmega_v3" in r.presets
+
+
+# ---------------------------------------------------------------------------
+# Dispatcher integration
+# ---------------------------------------------------------------------------
+
+
+def test_dispatch_tokenizer_encode_visualize_round_trip():
+    resp = dispatch({
+        "jsonrpc": "2.0", "id": "t1",
+        "method": "tokenizer.encode_visualize",
+        "params": {"tokenizer_source": _VENDORED, "text": "abc"},
+    })
+    assert resp.error is None
+    assert resp.result["token_count"] >= 1
+
+
+def test_dispatch_tokenizer_list_presets_round_trip():
+    resp = dispatch({
+        "jsonrpc": "2.0", "id": "lp", "method": "tokenizer.list_presets",
+    })
+    assert resp.error is None
+    assert "presets" in resp.result
+    assert len(resp.result["presets"]) >= 12
+
+
+def test_dispatch_tokenizer_missing_source_returns_invalid_params():
+    resp = dispatch({
+        "jsonrpc": "2.0", "id": "tx",
+        "method": "tokenizer.encode_visualize",
+        "params": {"tokenizer_source": "/nope.json", "text": "x"},
+    })
+    # FileNotFoundError surfaces as INTERNAL_ERROR (not InvalidParams)
+    # because it's a runtime failure, not a schema failure.
+    assert resp.error is not None
+    assert resp.error.code in (-32603, -32602)
+
+
+def test_method_registry_includes_tokenizer_endpoints():
+    from cppmega_v4.jsonrpc import METHOD_REGISTRY
+    assert "tokenizer.encode_visualize" in METHOD_REGISTRY
+    assert "tokenizer.list_presets" in METHOD_REGISTRY