|
| 1 | +// V7-F55 — preset × tokenizer compatibility matrix. |
| 2 | +// Each cell shows a status pill driven by a real tokenizer.encode_visualize |
| 3 | +// roundtrip. Cells start in 'idle'; user clicks "Probe all" to fill |
| 4 | +// the grid (sequential, ~5-50ms each) or clicks individual cells to |
| 5 | +// fetch one. Click any populated cell to expand the inline panel with |
| 6 | +// the first 10 token ids it produced. |
| 7 | + |
| 8 | +import { useState } from "react"; |
| 9 | +import type { RpcClient } from "@/lib/rpc"; |
| 10 | +import { HelpIcon } from "@/components/HelpIcon"; |
| 11 | + |
| 12 | +const PROBE_TEXT = "def hello():\n print('hello, world')\n"; |
| 13 | + |
| 14 | +export interface TokenizerMatrixTabProps { |
| 15 | + rpc: RpcClient | null; |
| 16 | + presets: readonly string[]; |
| 17 | + tokenizers: readonly string[]; |
| 18 | +} |
| 19 | + |
| 20 | +type Status = "idle" | "ok" | "incompat" | "error"; |
| 21 | + |
| 22 | +interface CellState { |
| 23 | + status: Status; |
| 24 | + token_count?: number; |
| 25 | + bytes_per_token_avg?: number; |
| 26 | + first_token_ids?: number[]; |
| 27 | + error?: string; |
| 28 | +} |
| 29 | + |
| 30 | +function cellKey(preset: string, tokenizer: string): string { |
| 31 | + return `${preset}::${tokenizer}`; |
| 32 | +} |
| 33 | + |
| 34 | +const PILL: Record<Status, { bg: string; fg: string; label: string }> = { |
| 35 | + idle: { bg: "#f3f4f6", fg: "#6b7280", label: "—" }, |
| 36 | + ok: { bg: "#dcfce7", fg: "#166534", label: "ok" }, |
| 37 | + incompat: { bg: "#fef3c7", fg: "#92400e", label: "incompat" }, |
| 38 | + error: { bg: "#fee2e2", fg: "#991b1b", label: "error" }, |
| 39 | +}; |
| 40 | + |
| 41 | +export function TokenizerMatrixTab({ |
| 42 | + rpc, presets, tokenizers, |
| 43 | +}: TokenizerMatrixTabProps): JSX.Element { |
| 44 | + const [cells, setCells] = useState<Map<string, CellState>>(new Map()); |
| 45 | + const [running, setRunning] = useState(false); |
| 46 | + const [expanded, setExpanded] = useState<string | null>(null); |
| 47 | + |
| 48 | + async function probeOne(preset: string, tokenizer: string) { |
| 49 | + setCells((prev) => { |
| 50 | + const next = new Map(prev); |
| 51 | + next.set(cellKey(preset, tokenizer), { status: "idle" }); |
| 52 | + return next; |
| 53 | + }); |
| 54 | + try { |
| 55 | + if (!rpc) throw new Error("rpc unavailable"); |
| 56 | + const r = await rpc.call<{ |
| 57 | + tokens: { id: number }[]; |
| 58 | + token_count: number; |
| 59 | + bytes_per_token_avg: number; |
| 60 | + }>("tokenizer.encode_visualize", { |
| 61 | + tokenizer_source: tokenizer, |
| 62 | + text: PROBE_TEXT, |
| 63 | + }); |
| 64 | + const status: Status = r.token_count > 0 ? "ok" : "incompat"; |
| 65 | + const ids = r.tokens.slice(0, 10).map((t) => t.id); |
| 66 | + setCells((prev) => { |
| 67 | + const next = new Map(prev); |
| 68 | + next.set(cellKey(preset, tokenizer), { |
| 69 | + status, |
| 70 | + token_count: r.token_count, |
| 71 | + bytes_per_token_avg: r.bytes_per_token_avg, |
| 72 | + first_token_ids: ids, |
| 73 | + }); |
| 74 | + return next; |
| 75 | + }); |
| 76 | + } catch (e) { |
| 77 | + setCells((prev) => { |
| 78 | + const next = new Map(prev); |
| 79 | + next.set(cellKey(preset, tokenizer), { |
| 80 | + status: "error", error: (e as Error).message, |
| 81 | + }); |
| 82 | + return next; |
| 83 | + }); |
| 84 | + } |
| 85 | + } |
| 86 | + |
| 87 | + async function probeAll() { |
| 88 | + setRunning(true); |
| 89 | + for (const p of presets) { |
| 90 | + for (const t of tokenizers) { |
| 91 | + await probeOne(p, t); |
| 92 | + } |
| 93 | + } |
| 94 | + setRunning(false); |
| 95 | + } |
| 96 | + |
| 97 | + return ( |
| 98 | + <div data-testid="tokenizer-matrix-tab" |
| 99 | + style={{ padding: 12, fontFamily: "system-ui, sans-serif", |
| 100 | + fontSize: 12, overflowY: "auto", flex: 1 }}> |
| 101 | + <div style={{ display: "flex", alignItems: "center", gap: 8, |
| 102 | + marginBottom: 8 }}> |
| 103 | + <h3 style={{ margin: 0, fontSize: 14 }}> |
| 104 | + Tokenizer × preset compatibility matrix |
| 105 | + </h3> |
| 106 | + <HelpIcon topic="tokenizer_matrix" /> |
| 107 | + <button data-testid="tokmatrix-probe-all" |
| 108 | + disabled={running} |
| 109 | + onClick={probeAll} |
| 110 | + style={{ padding: "4px 10px", border: "none", |
| 111 | + borderRadius: 4, |
| 112 | + background: running ? "#e5e7eb" : "#2563eb", |
| 113 | + color: running ? "#6b7280" : "white", |
| 114 | + cursor: running ? "wait" : "pointer" }}> |
| 115 | + {running ? "Probing…" : "Probe all cells"} |
| 116 | + </button> |
| 117 | + </div> |
| 118 | + <table data-testid="tokenizer-matrix-table" |
| 119 | + style={{ borderCollapse: "collapse", width: "100%" }}> |
| 120 | + <thead> |
| 121 | + <tr style={{ borderBottom: "1px solid #e5e7eb" }}> |
| 122 | + <th style={th}>preset \ tokenizer</th> |
| 123 | + {tokenizers.map((t) => ( |
| 124 | + <th key={t} style={th} |
| 125 | + title={t} |
| 126 | + data-testid={`tokmatrix-col-${tokFileLabel(t)}`}> |
| 127 | + {tokFileLabel(t)} |
| 128 | + </th> |
| 129 | + ))} |
| 130 | + </tr> |
| 131 | + </thead> |
| 132 | + <tbody> |
| 133 | + {presets.map((p) => ( |
| 134 | + <tr key={p} data-testid={`tokmatrix-row-${p}`} |
| 135 | + style={{ borderBottom: "1px solid #f3f4f6" }}> |
| 136 | + <td style={{ ...td, fontWeight: 600 }}>{p}</td> |
| 137 | + {tokenizers.map((t) => { |
| 138 | + const key = cellKey(p, t); |
| 139 | + const c = cells.get(key) ?? { status: "idle" as Status }; |
| 140 | + const pill = PILL[c.status]; |
| 141 | + const isExpanded = expanded === key; |
| 142 | + const tokLabel = tokFileLabel(t); |
| 143 | + return ( |
| 144 | + <td key={t} style={td} |
| 145 | + data-testid={`tokmatrix-${p}-${tokLabel}`} |
| 146 | + data-status={c.status}> |
| 147 | + <button |
| 148 | + data-testid={`tokmatrix-${p}-${tokLabel}-pill`} |
| 149 | + onClick={() => { |
| 150 | + if (c.status === "idle") probeOne(p, t); |
| 151 | + else setExpanded(isExpanded ? null : key); |
| 152 | + }} |
| 153 | + style={{ background: pill.bg, color: pill.fg, |
| 154 | + border: "none", padding: "2px 8px", |
| 155 | + borderRadius: 9999, cursor: "pointer", |
| 156 | + fontWeight: 600, fontSize: 11 }}> |
| 157 | + {pill.label} |
| 158 | + </button> |
| 159 | + {isExpanded && c.first_token_ids && ( |
| 160 | + <div data-testid={`tokmatrix-${p}-${tokLabel}-expand`} |
| 161 | + style={{ marginTop: 4, fontFamily: "monospace", |
| 162 | + fontSize: 11, color: "#374151" }}> |
| 163 | + ids: [{c.first_token_ids.join(", ")}] |
| 164 | + <br /> |
| 165 | + count: {c.token_count}, bpt: {c.bytes_per_token_avg?.toFixed(2)} |
| 166 | + </div> |
| 167 | + )} |
| 168 | + {isExpanded && c.error && ( |
| 169 | + <div data-testid={`tokmatrix-${p}-${tokLabel}-expand-error`} |
| 170 | + style={{ marginTop: 4, color: "#991b1b", |
| 171 | + fontSize: 11 }}> |
| 172 | + {c.error} |
| 173 | + </div> |
| 174 | + )} |
| 175 | + </td> |
| 176 | + ); |
| 177 | + })} |
| 178 | + </tr> |
| 179 | + ))} |
| 180 | + </tbody> |
| 181 | + </table> |
| 182 | + </div> |
| 183 | + ); |
| 184 | +} |
| 185 | + |
| 186 | +function tokFileLabel(path: string): string { |
| 187 | + // Strip directory + extension so cell testids stay readable. |
| 188 | + const base = path.split("/").pop() ?? path; |
| 189 | + return base.replace(/\.json$/, ""); |
| 190 | +} |
| 191 | + |
| 192 | +const th: React.CSSProperties = { |
| 193 | + textAlign: "left", padding: "4px 6px", |
| 194 | + color: "#374151", fontWeight: 600, |
| 195 | +}; |
| 196 | +const td: React.CSSProperties = { padding: "4px 6px", |
| 197 | + verticalAlign: "top" }; |
0 commit comments