Skip to content

Commit 73b4715

Browse files
apstenku123apstenku123
committed
feat(v4-3): TokenizerPlayground → stage_options.train.tokenizer_path
Closes V4-3 / cppmega-mlx-9zp. Completes G2: V4-2 made backend tokenize; V4-3 lets the user pick the tokenizer through the UI. TokenizerPlayground: - new prop onUseForTrain(tokenizerSource) + trainTokenizerPath - per-panel button data-testid='tokenizer-use-for-train-{i}' next to Encode — disabled until source set; ✓ Train label when this is the active training tokenizer. App.tsx: - wires TokenizerPlayground → setTrainTokenizerPath (V4-1 added the state slot; this fills it). E2E 29_tokenizer_train_threading.spec.ts: full UI walk Data → Use parquet → Tokenizer → Use tokenizer → Train; asserts train-data-source indicator shows both basenames AND extras.data_source ∈ {parquet, parquet_tokenized}. 4 new vitest (hidden when prop absent / disabled when empty source / fires callback / ✓ Train label on active panel). Regression 177/177 vitest.
1 parent a2ce456 commit 73b4715

4 files changed

Lines changed: 126 additions & 2 deletions

File tree

‎vbgui/e2e/scenarios/29_tokenizer_train_threading.spec.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// V4-3: Tokenizer Playground → stage_options.train.tokenizer_path
2+
// threading. Combined with V4-1 parquet path, this proves the full
3+
// real-data path: UI parquet + UI tokenizer → stage_train_tokenized.
4+
5+
import { test, expect } from "@playwright/test";
6+
import { gotoApp, selectPreset, clickTab, closeModal } from "../fixtures";
7+
import { loadMatrix } from "../utils/matrix";
8+
9+
test("V4-3: Tokenizer Playground tokenizer reaches stage_train tokenizer_path",
10+
async ({ page }) => {
11+
test.setTimeout(120_000);
12+
const matrix = loadMatrix();
13+
const parquet = matrix.parquets.T2_gpt2_small__P1_minimal.path;
14+
const tokenizer = matrix.tokenizers.T2_gpt2_small.path;
15+
16+
await gotoApp(page);
17+
await selectPreset(page, "llama3_8b");
18+
19+
// Load parquet + Use-for-train
20+
await clickTab(page, "data");
21+
await page.getByTestId("data-inspector").waitFor();
22+
await page.getByTestId("data-path").fill(parquet);
23+
await page.getByTestId("data-load").click();
24+
await page.getByTestId("data-metrics").waitFor({ timeout: 8_000 });
25+
await page.getByTestId("data-use-for-train").click();
26+
27+
// Pick a tokenizer in Playground + Use-for-train
28+
await clickTab(page, "tokenizer");
29+
await page.getByTestId("tokenizer-playground").waitFor();
30+
await page.getByTestId("add-panel").click();
31+
await page.getByTestId("tokenizer-source-0").fill(tokenizer);
32+
await page.getByTestId("tokenizer-encode-0").click();
33+
await page.getByTestId("tokenizer-metrics-0").waitFor({ timeout: 8_000 });
34+
await page.getByTestId("tokenizer-use-for-train-0").click();
35+
36+
// Indicator should now show both parquet + tokenizer
37+
const indicator = page.getByTestId("train-data-source");
38+
await expect(indicator).toContainText("parquet:");
39+
await expect(indicator).toContainText("tok:");
40+
41+
// Train + assert backend received tokenizer_path
42+
await clickTab(page, "canvas");
43+
await page.getByTestId("run-pipeline-toggle").click();
44+
await page.getByTestId("run-pipeline-train").click();
45+
const modal = page.getByTestId("run-result-modal");
46+
await modal.waitFor({ timeout: 60_000 });
47+
await page.getByTestId("run-result-expand-train").click();
48+
49+
// The T2 parquet has only input_ids (no 'text' column), so V4-2
50+
// tokenizer path falls through to V3-2 raw-int path. Either way
51+
// data_source must NOT be 'synthetic'.
52+
const dataSource = await page.getByTestId(
53+
"run-result-extras-train-data_source").textContent();
54+
expect(["parquet", "parquet_tokenized"]).toContain(dataSource?.trim());
55+
56+
await closeModal(page);
57+
});

‎vbgui/src/App.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,9 @@ export function App(): JSX.Element {
407407
</>
408408
)}
409409
{activeTab === "tokenizer" && (
410-
<TokenizerPlayground rpc={rpc} />
410+
<TokenizerPlayground rpc={rpc}
411+
onUseForTrain={(t) => setTrainTokenizerPath(t)}
412+
trainTokenizerPath={trainTokenizerPath} />
411413
)}
412414
{activeTab === "data" && (
413415
<DataInspector rpc={rpc}

‎vbgui/src/components/TokenizerPlayground.tsx‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@ export interface TokenizerPlaygroundProps {
3535
rpc: RpcClient;
3636
initialSources?: string[]; // up to 3
3737
maxPanels?: number; // default 3
38+
/** V4-3: callback when user picks this tokenizer for training.
39+
* App stores in trainTokenizerPath; handleRunPipeline forwards via
40+
* stage_options.train.tokenizer_path so backend V4-2 path can
41+
* tokenize parquet text. */
42+
onUseForTrain?: (tokenizerSource: string) => void;
43+
/** V4-3: current path App is using for training (drives ✓ label). */
44+
trainTokenizerPath?: string | null;
3845
}
3946

4047
const COLORS = ["#fde68a", "#bfdbfe", "#bbf7d0", "#fecaca", "#ddd6fe",
@@ -47,6 +54,7 @@ function colorForId(id: number, isSpecial: boolean): string {
4754

4855
export function TokenizerPlayground({
4956
rpc, initialSources = [], maxPanels = 3,
57+
onUseForTrain, trainTokenizerPath,
5058
}: TokenizerPlaygroundProps): JSX.Element {
5159
const [text, setText] = useState("Hello, world!\ndef foo():\n return 42");
5260
const [hoverSpan, setHoverSpan] = useState<{ start: number; end: number } | null>(null);
@@ -113,7 +121,9 @@ export function TokenizerPlayground({
113121
onSourceChange={(s) => setSource(i, s)}
114122
onEncode={() => runEncode(i)}
115123
onRemove={() => removePanel(i)}
116-
onHover={setHoverSpan} />
124+
onHover={setHoverSpan}
125+
onUseForTrain={onUseForTrain}
126+
trainTokenizerPath={trainTokenizerPath} />
117127
))}
118128
</div>
119129
</div>
@@ -129,10 +139,13 @@ interface TokenizerPanelProps {
129139
onEncode: () => void;
130140
onRemove: () => void;
131141
onHover: (span: { start: number; end: number } | null) => void;
142+
onUseForTrain?: (tokenizerSource: string) => void;
143+
trainTokenizerPath?: string | null;
132144
}
133145

134146
function TokenizerPanel({
135147
index, state, hoverSpan, onSourceChange, onEncode, onRemove, onHover,
148+
onUseForTrain, trainTokenizerPath,
136149
}: TokenizerPanelProps): JSX.Element {
137150
return (
138151
<section data-testid={`tokenizer-panel-${index}`}
@@ -148,6 +161,22 @@ function TokenizerPanel({
148161
<button data-testid={`tokenizer-encode-${index}`} onClick={onEncode}>
149162
Encode
150163
</button>
164+
{onUseForTrain && (
165+
<button data-testid={`tokenizer-use-for-train-${index}`}
166+
disabled={!state.source}
167+
title={trainTokenizerPath === state.source
168+
? "Currently used for training"
169+
: "Send this tokenizer to stage_train"}
170+
onClick={() => onUseForTrain(state.source)}
171+
style={{
172+
background: trainTokenizerPath === state.source
173+
? "#dcfce7" : undefined,
174+
color: trainTokenizerPath === state.source
175+
? "#166534" : undefined,
176+
}}>
177+
{trainTokenizerPath === state.source ? "✓ Train" : "→ Train"}
178+
</button>
179+
)}
151180
<button data-testid={`tokenizer-remove-${index}`} onClick={onRemove}>×</button>
152181
</div>
153182

‎vbgui/tests/TokenizerPlayground.test.tsx‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,42 @@ describe("TokenizerPlayground", () => {
7171
expect(screen.getByTestId("tokenizer-panel-0")).toBeTruthy();
7272
});
7373

74+
// V4-3: Use-for-train button
75+
it("tokenizer-use-for-train-{i} hidden when onUseForTrain absent", () => {
76+
render(<TokenizerPlayground rpc={mockClient(SAMPLE_RESULT)}
77+
initialSources={["/x.json"]} />);
78+
expect(screen.queryByTestId("tokenizer-use-for-train-0")).toBeNull();
79+
});
80+
81+
it("tokenizer-use-for-train-{i} disabled until source is set", () => {
82+
const onUseForTrain = vi.fn();
83+
render(<TokenizerPlayground rpc={mockClient(SAMPLE_RESULT)}
84+
initialSources={[""]}
85+
onUseForTrain={onUseForTrain} />);
86+
expect(screen.getByTestId("tokenizer-use-for-train-0")
87+
.hasAttribute("disabled")).toBe(true);
88+
});
89+
90+
it("tokenizer-use-for-train-{i} fires onUseForTrain(source)", () => {
91+
const onUseForTrain = vi.fn();
92+
render(<TokenizerPlayground rpc={mockClient(SAMPLE_RESULT)}
93+
initialSources={["/my/tok.json"]}
94+
onUseForTrain={onUseForTrain} />);
95+
fireEvent.click(screen.getByTestId("tokenizer-use-for-train-0"));
96+
expect(onUseForTrain).toHaveBeenCalledWith("/my/tok.json");
97+
});
98+
99+
it("active panel shows ✓ Train when trainTokenizerPath matches", () => {
100+
render(<TokenizerPlayground rpc={mockClient(SAMPLE_RESULT)}
101+
initialSources={["/active.json", "/other.json"]}
102+
onUseForTrain={() => {}}
103+
trainTokenizerPath="/active.json" />);
104+
expect(screen.getByTestId("tokenizer-use-for-train-0").textContent)
105+
.toContain("✓ Train");
106+
expect(screen.getByTestId("tokenizer-use-for-train-1").textContent)
107+
.not.toContain("✓ Train");
108+
});
109+
74110
it("Renders error envelope when backend fails", async () => {
75111
const failing = new RpcClient({
76112
baseUrl: "http://x",

0 commit comments

Comments
 (0)