feat(v4-1): DataInspector → stage_options.train.parquet_path threading

apstenku123 · apstenku123 · commit ee094dd705f9 · 2026-05-21T22:59:29.000+02:00
Closes V4-1 / cppmega-mlx-vsw. Fixes G1: V3-2 plumbed the backend to
accept opts.parquet_path but the DataInspector selection never reached
it, so user-picked parquet shards stayed decorative for training.

DataInspector:
  - new prop onUseForTrain(parquetPath, tokenizerPath | null)
  - new prop trainParquetPath (drives ✓ Training label)
  - data-testid='data-use-for-train' button next to Load — disabled
    until a parquet has been loaded.

App.tsx:
  - new state trainParquetPath / trainTokenizerPath
  - handleRunPipeline forwards both via stage_options.train.{parquet_path,
    tokenizer_path} when set (tokenizer path lands ahead of V4-2 backend
    consumer — harmless ignored extra now).

TopBar:
  - new props trainParquetPath / trainTokenizerPath
  - data-testid='train-data-source' indicator reads 'synthetic' by
    default; upgrades to 'parquet: &lt;basename&gt;' or
    'parquet: ... · tok: ...' when set.

New e2e 28_parquet_train_threading.spec.ts (2 cells):
  + positive: load fixture parquet → Use-for-train → Train → assert
    extras.data_source === "parquet" + token_count &gt; 0
  + negative: skip Use-for-train → Train → extras.data_source ===
    "synthetic" (V3-2 fallback still works)

7 new vitest (3 TopBar + 4 DataInspector). Regression 173/173 + 8/8
adjacent e2e (11_ui_to_train + 15_gating).
diff --git a/vbgui/e2e/scenarios/28_parquet_train_threading.spec.ts b/vbgui/e2e/scenarios/28_parquet_train_threading.spec.ts
@@ -0,0 +1,81 @@
+// V4-1: DataInspector → stage_options.train.parquet_path threading.
+//
+// Walks: load parquet in Data tab → click "Use for training" → switch
+// to canvas → Train → assert extras.data_source === "parquet" AND
+// extras.token_count > 0 (proves UI selection actually reached backend).
+//
+// Closes G1 from V4 audit (UI parquet decorative for training).
+
+import { test, expect } from "@playwright/test";
+import { gotoApp, selectPreset, clickTab, closeModal } from "../fixtures";
+import { loadMatrix } from "../utils/matrix";
+import { readTrainExtras } from "../utils/train_extras";
+
+test("V4-1: DataInspector parquet reaches stage_train via stage_options",
+  async ({ page }) => {
+    test.setTimeout(120_000);
+    const matrix = loadMatrix();
+    const parquet = matrix.parquets.T2_gpt2_small__P1_minimal.path;
+
+    await gotoApp(page);
+    await selectPreset(page, "llama3_8b");
+
+    // Default indicator: synthetic
+    await expect(page.getByTestId("train-data-source")).toContainText(
+      "synthetic");
+
+    // Load parquet in Data tab
+    await clickTab(page, "data");
+    await page.getByTestId("data-inspector").waitFor();
+    await page.getByTestId("data-path").fill(parquet);
+    await page.getByTestId("data-load").click();
+    await page.getByTestId("data-metrics").waitFor({ timeout: 8_000 });
+
+    // Use this parquet for training
+    await page.getByTestId("data-use-for-train").click();
+
+    // Indicator now shows parquet basename
+    const indicator = page.getByTestId("train-data-source");
+    await expect(indicator).toContainText("parquet:");
+    await expect(indicator).toContainText("T2_gpt2_small__P1_minimal");
+
+    // Train
+    await clickTab(page, "canvas");
+    await page.getByTestId("run-pipeline-toggle").click();
+    await page.getByTestId("run-pipeline-train").click();
+    const modal = page.getByTestId("run-result-modal");
+    await modal.waitFor({ timeout: 60_000 });
+
+    const extras = await readTrainExtras(page);
+    // Direct DOM check on new data_source / token_count primitives.
+    const dataSource = await page.getByTestId(
+      "run-result-extras-train-data_source").textContent();
+    expect(dataSource?.trim()).toBe("parquet");
+    const tokenCount = parseInt(
+      (await page.getByTestId(
+        "run-result-extras-train-token_count").textContent()) ?? "0", 10);
+    expect(tokenCount).toBeGreaterThan(0);
+    // Weights moved
+    expect(extras.weight_delta_norm).toBeGreaterThan(0);
+
+    await closeModal(page);
+  });
+
+test("V4-1 negative: no Use-for-train click → stage_train falls back to synthetic",
+  async ({ page }) => {
+    test.setTimeout(120_000);
+    await gotoApp(page);
+    await selectPreset(page, "llama3_8b");
+
+    // Don't click Use-for-train; just run Train.
+    await page.getByTestId("run-pipeline-toggle").click();
+    await page.getByTestId("run-pipeline-train").click();
+    const modal = page.getByTestId("run-result-modal");
+    await modal.waitFor({ timeout: 60_000 });
+
+    await page.getByTestId("run-result-expand-train").click();
+    const dataSource = await page.getByTestId(
+      "run-result-extras-train-data_source").textContent();
+    expect(dataSource?.trim()).toBe("synthetic");
+    await closeModal(page);
+  });
diff --git a/vbgui/src/App.tsx b/vbgui/src/App.tsx
@@ -97,6 +97,12 @@ export function App(): JSX.Element {
     { brick: string; param: string; value: unknown;
       source: "user" | "auto"; reason: string }[]
   >([]);
+  // V4-1: data source for stage_train. When set, handleRunPipeline
+  // forwards via stage_options.train.parquet_path; UI shows indicator.
+  const [trainParquetPath, setTrainParquetPath] =
+    useState<string | null>(null);
+  const [trainTokenizerPath, setTrainTokenizerPath] =
+    useState<string | null>(null);
 
   const rpc = useRpc({
     baseUrl: (import.meta.env.VITE_BACKEND_URL as string | undefined)
@@ -275,9 +281,16 @@ export function App(): JSX.Element {
     const stages = mode === "smoke" ? SMOKE_STAGES
                  : mode === "full"  ? FULL_STAGES : TRAIN_STAGES;
     // V3-6: TopBar exposes train_num_steps; thread it via stage_options.
+    // V4-1: forward parquet_path + tokenizer_path picked in Data/Tokenizer tabs.
     const stage_options: Record<string, Record<string, unknown>> = {};
-    if (mode === "train" && typeof opts?.num_steps === "number") {
-      stage_options.train = { num_steps: opts.num_steps };
+    if (mode === "train") {
+      const trainOpts: Record<string, unknown> = {};
+      if (typeof opts?.num_steps === "number") {
+        trainOpts.num_steps = opts.num_steps;
+      }
+      if (trainParquetPath) trainOpts.parquet_path = trainParquetPath;
+      if (trainTokenizerPath) trainOpts.tokenizer_path = trainTokenizerPath;
+      if (Object.keys(trainOpts).length > 0) stage_options.train = trainOpts;
     }
     try {
       const r = await rpc.call<RunReport>("pipeline.run", {
@@ -288,7 +301,7 @@ export function App(): JSX.Element {
     } catch (e) {
       setRunError(String(e));
     }
-  }, [rpc]);
+  }, [rpc, trainParquetPath, trainTokenizerPath]);
 
   const handleShardingAccept = useCallback((idx: number) => {
     const chosen = proposals[idx];
@@ -317,6 +330,8 @@ export function App(): JSX.Element {
           onCompileModeChange={(m) => dispatch({ type: "sharding.set",
             sharding: { ...spec.sharding, compile_mode: m } })}
           onRunPipeline={handleRunPipeline}
+          trainParquetPath={trainParquetPath}
+          trainTokenizerPath={trainTokenizerPath}
           trainDisabled={
             (() => {
               // V3-8/V3-9: gate Train on gotcha severity. The verify RPC
@@ -395,7 +410,12 @@ export function App(): JSX.Element {
             <TokenizerPlayground rpc={rpc} />
           )}
           {activeTab === "data" && (
-            <DataInspector rpc={rpc} />
+            <DataInspector rpc={rpc}
+              onUseForTrain={(p, t) => {
+                setTrainParquetPath(p);
+                if (t !== null) setTrainTokenizerPath(t);
+              }}
+              trainParquetPath={trainParquetPath} />
           )}
         </div>
         <BottomStrip state={spec} fusedRegionCount={0} />
diff --git a/vbgui/src/components/DataInspector.tsx b/vbgui/src/components/DataInspector.tsx
@@ -22,6 +22,11 @@ export interface DataInspectorProps {
   rpc: RpcClient;
   initialPath?: string;
   pageSize?: number;
+  /** V4-1: callback when user picks the loaded parquet for training.
+   *  App stores the path and forwards via stage_options.train.parquet_path. */
+  onUseForTrain?: (parquetPath: string, tokenizerPath: string | null) => void;
+  /** V4-1: current path App is using for training (drives button label). */
+  trainParquetPath?: string | null;
 }
 
 const CHANNEL_COLORS = ["#fde68a", "#bfdbfe", "#bbf7d0", "#fecaca",
@@ -56,6 +61,7 @@ interface RoundtripCheckResult {
 
 export function DataInspector({
   rpc, initialPath = "", pageSize = 16,
+  onUseForTrain, trainParquetPath,
 }: DataInspectorProps): JSX.Element {
   const [path, setPath] = useState(initialPath);
   const [offset, setOffset] = useState(0);
@@ -118,6 +124,21 @@ export function DataInspector({
         <button data-testid="data-load" onClick={() => load(0)}>
           Load
         </button>
+        <button data-testid="data-use-for-train"
+                disabled={!result || !onUseForTrain}
+                title={trainParquetPath === path
+                  ? "Currently used for training"
+                  : "Send this parquet (and tokenizer if set) to stage_train"}
+                onClick={() => onUseForTrain?.(path,
+                  tokenizerSource || null)}
+                style={{
+                  background: trainParquetPath === path
+                    ? "#dcfce7" : undefined,
+                  color: trainParquetPath === path
+                    ? "#166534" : undefined,
+                }}>
+          {trainParquetPath === path ? "✓ Training" : "Use for training"}
+        </button>
       </header>
       <header style={{ display: "flex", gap: 8, alignItems: "center" }}>
         <span style={{ fontSize: 11, color: "#6b7280" }}>Tokenizer:</span>
diff --git a/vbgui/src/components/TopBar.tsx b/vbgui/src/components/TopBar.tsx
@@ -17,6 +17,12 @@ export interface TopBarProps {
   /** V3-8/V3-9: when present, Train button is rendered disabled with
    *  reason exposed via data-testid='top-bar-train-disabled-reason'. */
   trainDisabled?: { reason: string } | null;
+  /** V4-1: parquet + tokenizer paths picked in Data/Tokenizer tabs.
+   *  Drives the data-testid='train-data-source' indicator next to
+   *  Train so the user can tell whether training will use real tokens
+   *  or fall back to synthetic. */
+  trainParquetPath?: string | null;
+  trainTokenizerPath?: string | null;
 }
 
 export function TopBar(p: TopBarProps): JSX.Element {
@@ -59,6 +65,17 @@ export function TopBar(p: TopBarProps): JSX.Element {
 
       <MemoryBar state={p.state} />
 
+      <span data-testid="train-data-source"
+            style={{ fontSize: 10,
+                     color: p.trainParquetPath ? "#16a34a" : "#9ca3af",
+                     fontFamily: "monospace" }}>
+        {p.trainParquetPath
+          ? `parquet: ${basename(p.trainParquetPath)}` +
+            (p.trainTokenizerPath
+              ? ` · tok: ${basename(p.trainTokenizerPath)}` : "")
+          : "synthetic"}
+      </span>
+
       <div style={{ position: "relative" }}>
         <button data-testid="run-pipeline"
                 onClick={() => p.onRunPipeline("smoke")}>
@@ -113,6 +130,11 @@ export function TopBar(p: TopBarProps): JSX.Element {
   );
 }
 
+function basename(p: string): string {
+  const slash = p.lastIndexOf("/");
+  return slash >= 0 ? p.slice(slash + 1) : p;
+}
+
 const menuItem: React.CSSProperties = {
   display: "block", padding: "6px 12px", border: "none",
   background: "white", cursor: "pointer", textAlign: "left", width: "100%",
diff --git a/vbgui/tests/DataInspector.test.tsx b/vbgui/tests/DataInspector.test.tsx
@@ -91,6 +91,54 @@ describe("DataInspector", () => {
     });
   });
 
+  // V4-1: Use-for-train button
+  it("data-use-for-train disabled until parquet loaded", () => {
+    render(<DataInspector rpc={mockClient(SAMPLE)} initialPath="/x"
+                          onUseForTrain={() => {}} />);
+    expect(screen.getByTestId("data-use-for-train")
+      .hasAttribute("disabled")).toBe(true);
+  });
+
+  it("data-use-for-train fires onUseForTrain(path, null) after load",
+    async () => {
+      const onUseForTrain = vi.fn();
+      render(<DataInspector rpc={mockClient(SAMPLE)}
+                            initialPath="/tmp/sh.parquet"
+                            onUseForTrain={onUseForTrain} />);
+      fireEvent.click(screen.getByTestId("data-load"));
+      await waitFor(() => {
+        expect(screen.getByTestId("data-metrics")).toBeTruthy();
+      });
+      fireEvent.click(screen.getByTestId("data-use-for-train"));
+      expect(onUseForTrain).toHaveBeenCalledWith("/tmp/sh.parquet", null);
+    });
+
+  it("data-use-for-train fires onUseForTrain(path, tokenizer) when both set",
+    async () => {
+      const onUseForTrain = vi.fn();
+      render(<DataInspector rpc={mockClient(SAMPLE)}
+                            initialPath="/p.parquet"
+                            onUseForTrain={onUseForTrain} />);
+      fireEvent.click(screen.getByTestId("data-load"));
+      await waitFor(() =>
+        expect(screen.getByTestId("data-metrics")).toBeTruthy());
+      fireEvent.change(screen.getByTestId("data-tokenizer-path"),
+        { target: { value: "/t.json" } });
+      fireEvent.click(screen.getByTestId("data-use-for-train"));
+      expect(onUseForTrain).toHaveBeenCalledWith("/p.parquet", "/t.json");
+    });
+
+  it("data-use-for-train shows ✓ Training when this path is active", async () => {
+    render(<DataInspector rpc={mockClient(SAMPLE)} initialPath="/p"
+                          trainParquetPath="/p"
+                          onUseForTrain={() => {}} />);
+    fireEvent.click(screen.getByTestId("data-load"));
+    await waitFor(() =>
+      expect(screen.getByTestId("data-metrics")).toBeTruthy());
+    expect(screen.getByTestId("data-use-for-train").textContent)
+      .toContain("Training");
+  });
+
   it("renders error envelope when backend fails", async () => {
     const failing = new RpcClient({
       baseUrl: "http://x",
diff --git a/vbgui/tests/TopBar.test.tsx b/vbgui/tests/TopBar.test.tsx
@@ -34,6 +34,35 @@ describe("TopBar", () => {
     expect(screen.getByTestId("run-pipeline")).toBeTruthy();
   });
 
+  // V4-1: train-data-source indicator
+  it("train-data-source reads 'synthetic' when no parquet selected", () => {
+    render(<TopBar {...defaultTopProps()} />);
+    expect(screen.getByTestId("train-data-source").textContent)
+      .toBe("synthetic");
+  });
+
+  it("train-data-source shows parquet basename when trainParquetPath set",
+    () => {
+      render(<TopBar {...defaultTopProps({
+        trainParquetPath: "/tmp/fixtures/T2_gpt2_small__P1.parquet",
+      })} />);
+      const indicator = screen.getByTestId("train-data-source").textContent!;
+      expect(indicator).toContain("parquet:");
+      expect(indicator).toContain("T2_gpt2_small__P1.parquet");
+      expect(indicator).not.toContain("/tmp/");
+    });
+
+  it("train-data-source appends tokenizer basename when both set", () => {
+    render(<TopBar {...defaultTopProps({
+      trainParquetPath: "/a/b/foo.parquet",
+      trainTokenizerPath: "/x/y/cppmega_tokenizer.json",
+    })} />);
+    const t = screen.getByTestId("train-data-source").textContent!;
+    expect(t).toContain("foo.parquet");
+    expect(t).toContain("tok:");
+    expect(t).toContain("cppmega_tokenizer.json");
+  });
+
   it("preset launcher fires onPresetDrop when chosen", () => {
     const onPresetDrop = vi.fn();
     render(<TopBar {...defaultTopProps({ onPresetDrop })} />);
