Skip to content

Commit 761d413

Browse files
committed
feat(v7-k1): tokenizer preset dropdown via tokenizer.list_presets RPC
Honest-closure: backend exposed PRESET_LIBRARY of 12 tokenizers through tokenizer.list_presets, but the playground asked users to type the source path by hand — preset names were 'hardcoded knowledge' nowhere surfaced. - TokenizerPlayground: fetches presets on mount and renders both an HTML <datalist> (auto-complete on the text input) and a per-panel <select> dropdown. Selecting an option populates the source input; the option list reflects backend state, so changes to PRESET_LIBRARY propagate without a frontend rebuild. - Falls back gracefully (no dropdown) if list_presets RPC errors.
1 parent 02ed118 commit 761d413

1 file changed

Lines changed: 40 additions & 3 deletions

File tree

vbgui/src/components/TokenizerPlayground.tsx

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { useCallback, useState } from "react";
1+
import { useCallback, useEffect, useState } from "react";
22
import type { RpcClient } from "@/lib/rpc";
33

44
export interface TokenSpan {
@@ -61,6 +61,21 @@ export function TokenizerPlayground({
6161
const [panels, setPanels] = useState<TokenizerPanelState[]>(
6262
() => initialSources.slice(0, maxPanels).map((source) => ({ source })),
6363
);
64+
// V7-K1: backend-driven tokenizer preset list via tokenizer.list_presets.
65+
// Replaces the previously hardcoded suggestion set so the UI picks up
66+
// PRESET_LIBRARY changes from the backend without a frontend rebuild.
67+
const [presets, setPresets] = useState<string[]>([]);
68+
useEffect(() => {
69+
let cancelled = false;
70+
(async () => {
71+
try {
72+
const res = await rpc.call<{ presets: string[] }>(
73+
"tokenizer.list_presets", {});
74+
if (!cancelled) setPresets(res.presets ?? []);
75+
} catch { /* leave empty — datalist degrades gracefully */ }
76+
})();
77+
return () => { cancelled = true; };
78+
}, [rpc]);
6479

6580
const runEncode = useCallback(async (idx: number) => {
6681
const panel = panels[idx];
@@ -112,11 +127,17 @@ export function TokenizerPlayground({
112127
padding: 6, border: "1px solid #d1d5db", borderRadius: 4 }}
113128
/>
114129

130+
<datalist id="tokenizer-preset-suggestions"
131+
data-testid="tokenizer-preset-suggestions">
132+
{presets.map((p) => <option key={p} value={p} />)}
133+
</datalist>
134+
115135
<div style={{ display: "grid",
116136
gridTemplateColumns: `repeat(${Math.max(1, panels.length)}, 1fr)`,
117137
gap: 8, flex: 1, minHeight: 0 }}>
118138
{panels.map((p, i) => (
119139
<TokenizerPanel key={i} index={i} state={p}
140+
presets={presets}
120141
hoverSpan={hoverSpan}
121142
onSourceChange={(s) => setSource(i, s)}
122143
onEncode={() => runEncode(i)}
@@ -134,6 +155,7 @@ export function TokenizerPlayground({
134155
interface TokenizerPanelProps {
135156
index: number;
136157
state: TokenizerPanelState;
158+
presets?: string[];
137159
hoverSpan: { start: number; end: number } | null;
138160
onSourceChange: (s: string) => void;
139161
onEncode: () => void;
@@ -144,8 +166,8 @@ interface TokenizerPanelProps {
144166
}
145167

146168
function TokenizerPanel({
147-
index, state, hoverSpan, onSourceChange, onEncode, onRemove, onHover,
148-
onUseForTrain, trainTokenizerPath,
169+
index, state, presets, hoverSpan, onSourceChange, onEncode, onRemove,
170+
onHover, onUseForTrain, trainTokenizerPath,
149171
}: TokenizerPanelProps): JSX.Element {
150172
return (
151173
<section data-testid={`tokenizer-panel-${index}`}
@@ -156,8 +178,23 @@ function TokenizerPanel({
156178
<input data-testid={`tokenizer-source-${index}`}
157179
type="text" placeholder="tokenizer.json path or hub id"
158180
value={state.source}
181+
list="tokenizer-preset-suggestions"
159182
onChange={(e) => onSourceChange(e.target.value)}
160183
style={{ flex: 1, fontFamily: "monospace", fontSize: 11 }} />
184+
{presets && presets.length > 0 && (
185+
<select data-testid={`tokenizer-preset-picker-${index}`}
186+
value=""
187+
onChange={(e) => {
188+
if (e.target.value) onSourceChange(e.target.value);
189+
e.currentTarget.value = "";
190+
}}
191+
title="Pick a backend tokenizer preset"
192+
style={{ fontSize: 11, maxWidth: 110 }}>
193+
<option value="">presets…</option>
194+
{presets.map((p) =>
195+
<option key={p} value={p}>{p}</option>)}
196+
</select>
197+
)}
161198
<button data-testid={`tokenizer-encode-${index}`} onClick={onEncode}>
162199
Encode
163200
</button>

0 commit comments

Comments
 (0)