From 446d8c1319ebaac888c47ba45bddd17352aa5e08 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 18 Jun 2026 00:43:28 -0500 Subject: [PATCH] feat: add better default precision selection --- .../src/components/GlobalFilterContext.tsx | 79 ++++++-- .../app/src/lib/default-precisions.test.ts | 185 ++++++++++++++++++ packages/app/src/lib/default-precisions.ts | 118 +++++++++++ packages/app/src/lib/url-state.ts | 5 +- 4 files changed, 369 insertions(+), 18 deletions(-) create mode 100644 packages/app/src/lib/default-precisions.test.ts create mode 100644 packages/app/src/lib/default-precisions.ts diff --git a/packages/app/src/components/GlobalFilterContext.tsx b/packages/app/src/components/GlobalFilterContext.tsx index 62ae64ff..6e7afb0b 100644 --- a/packages/app/src/components/GlobalFilterContext.tsx +++ b/packages/app/src/components/GlobalFilterContext.tsx @@ -37,6 +37,7 @@ import { SEQUENCE_OPTIONS, } from '@/lib/data-mappings'; import { computeAutoSwitchDecision } from '@/lib/unofficial-run-auto-switch'; +import { countCurvesByPrecision, resolveEffectivePrecisions } from '@/lib/default-precisions'; import type { AvailabilityRow, WorkflowInfoResponse } from '@/lib/api'; interface RunInfo { @@ -149,17 +150,23 @@ export function GlobalFilterProvider({ return Sequence.EightK_OneK; }); - const [selectedPrecisions, setSelectedPrecisionsRaw] = useState(() => { - if (initialPrecisions && initialPrecisions.length > 0) { - const valid = initialPrecisions.filter((p) => - (PRECISION_OPTIONS as readonly string[]).includes(p), - ); - if (valid.length > 0) return valid; - } - return [Precision.FP4]; - }); + const initialValidPrecisions = useMemo( + () => + (initialPrecisions ?? []).filter((p) => (PRECISION_OPTIONS as readonly string[]).includes(p)), + [initialPrecisions], + ); + const [selectedPrecisions, setSelectedPrecisionsRaw] = useState(() => + initialValidPrecisions.length > 0 ? initialValidPrecisions : [Precision.FP4], + ); + // Whether the precision was chosen explicitly (seeded prop, URL `i_prec`, + // preset, or manual toggle). Until then we auto-pick the densest precision + // for the current model/sequence so FP8-heavy models don't open barren. + const [precisionExplicit, setPrecisionExplicit] = useState( + () => initialValidPrecisions.length > 0, + ); const setSelectedPrecisions = useCallback((precisions: string[]) => { setSelectedPrecisionsRaw(precisions); + setPrecisionExplicit(true); }, []); // ── Run date / run ID ───────────────────────────────────────────────────── @@ -198,7 +205,10 @@ export function GlobalFilterProvider({ const precs = urlPrec .split(',') .filter((p) => (PRECISION_OPTIONS as readonly string[]).includes(p)); - if (precs.length > 0) setSelectedPrecisionsRaw(precs); + if (precs.length > 0) { + setSelectedPrecisionsRaw(precs); + setPrecisionExplicit(true); + } } applyIfMatches('g_rundate', RUNDATE_RE, setSelectedRunDateBase); applyIfMatches('g_runid', RUNID_RE, setSelectedRunId); @@ -294,12 +304,44 @@ export function GlobalFilterProvider({ return merged.length > 0 ? merged : ['fp4']; }, [availabilityRows, modelRows, effectiveSequence, unofficialAvailable, selectedModel]); - // Synchronously validated precisions - const effectivePrecisions = useMemo(() => { - const valid = selectedPrecisions.filter((p) => availablePrecisions.includes(p)); - if (valid.length > 0) return valid; - return availablePrecisions.length > 0 ? [availablePrecisions[0]] : selectedPrecisions; - }, [selectedPrecisions, availablePrecisions]); + // Curve count per precision (distinct hw/framework/spec/disagg series) for the + // selected model + sequence — drives the auto default toward the densest one. + const precisionCurveCounts = useMemo( + () => + countCurvesByPrecision( + modelRows.filter((r) => islOslToSequence(r.isl, r.osl) === effectiveSequence), + ), + [modelRows, effectiveSequence], + ); + + // Precisions present in a loaded unofficial run for the current model + sequence. + const unofficialPrecisionsForSelection = useMemo( + () => + unofficialAvailable + .filter((a) => a.model === selectedModel && a.sequence === effectiveSequence) + .flatMap((a) => a.precisions), + [unofficialAvailable, selectedModel, effectiveSequence], + ); + + // Synchronously validated precisions. When the user hasn't explicitly chosen a + // precision, auto-pick the densest (see resolveEffectivePrecisions). + const effectivePrecisions = useMemo( + () => + resolveEffectivePrecisions({ + selectedPrecisions, + availablePrecisions, + curveCounts: precisionCurveCounts, + unofficialPrecisions: unofficialPrecisionsForSelection, + explicit: precisionExplicit, + }), + [ + selectedPrecisions, + availablePrecisions, + precisionCurveCounts, + unofficialPrecisionsForSelection, + precisionExplicit, + ], + ); // Dates available for selected model + sequence + precisions const availableDates = useMemo(() => { @@ -397,7 +439,9 @@ export function GlobalFilterProvider({ g_rundate: selectedRunDate, g_runid: selectedRunId, i_seq: effectiveSequence, - i_prec: effectivePrecisions.join(','), + // Only pin the precision in the URL once chosen explicitly; in auto mode + // leave it out so the link keeps following the per-model densest default. + i_prec: precisionExplicit ? effectivePrecisions.join(',') : undefined, }); }, [ selectedModel, @@ -405,6 +449,7 @@ export function GlobalFilterProvider({ selectedRunId, effectiveSequence, effectivePrecisions, + precisionExplicit, setUrlParams, ]); diff --git a/packages/app/src/lib/default-precisions.test.ts b/packages/app/src/lib/default-precisions.test.ts new file mode 100644 index 00000000..9d8a72ca --- /dev/null +++ b/packages/app/src/lib/default-precisions.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, it } from 'vitest'; + +import { + MIN_SOLE_DEFAULT_CURVES, + countCurvesByPrecision, + pickDefaultPrecisions, + resolveEffectivePrecisions, +} from './default-precisions'; + +function row(precision: string, hardware: string, framework = 'vllm', disagg = false) { + return { precision, hardware, framework, spec_method: 'none', disagg }; +} + +describe('countCurvesByPrecision', () => { + it('counts distinct (hardware, framework, spec_method, disagg) per precision', () => { + const counts = countCurvesByPrecision([ + row('fp8', 'b200'), + row('fp8', 'b300'), + row('fp8', 'b200'), // dup curve, not counted again + row('fp4', 'mi355x', 'atom'), + ]); + expect(counts).toEqual({ fp8: 2, fp4: 1 }); + }); + + it('treats disagg and non-disagg of the same hw as distinct curves', () => { + const counts = countCurvesByPrecision([ + row('fp8', 'b200', 'vllm', false), + row('fp8', 'b200', 'vllm', true), + ]); + expect(counts).toEqual({ fp8: 2 }); + }); + + it('returns {} for no rows', () => { + expect(countCurvesByPrecision([])).toEqual({}); + }); +}); + +describe('pickDefaultPrecisions', () => { + it('picks the single densest precision only when every precision clears the threshold', () => { + // dsr1 shape: both dense. + expect(pickDefaultPrecisions({ fp4: 23, fp8: 38 })).toEqual(['fp8']); + }); + + it('shows both when one precision is below the threshold (MiniMax M3 shape)', () => { + // fp4 barren (1 curve) next to a dense fp8 → surface both, not just fp8. + expect(pickDefaultPrecisions({ fp4: 1, fp8: 14 })).toEqual(['fp4', 'fp8']); + }); + + it('keeps fp4 when it is the densest and both clear the threshold (dsv4 shape)', () => { + expect(pickDefaultPrecisions({ fp4: 28, fp8: 5 })).toEqual(['fp4']); + }); + + it('breaks ties in favor of fp4 when both clear the threshold', () => { + expect(pickDefaultPrecisions({ fp4: 8, fp8: 8 })).toEqual(['fp4']); + }); + + it('breaks non-fp4 ties by canonical enum order', () => { + // fp8 precedes bf16 in PRECISION_OPTIONS. + expect(pickDefaultPrecisions({ bf16: 6, fp8: 6 })).toEqual(['fp8']); + }); + + it('surfaces all precisions (sorted) when any is below threshold', () => { + expect(pickDefaultPrecisions({ fp8: 3, fp4: 2 })).toEqual(['fp4', 'fp8']); + // llama70b shape: fp4 sparse (3), fp8 dense (8) → both. + expect(pickDefaultPrecisions({ fp4: 3, fp8: 8 })).toEqual(['fp4', 'fp8']); + expect(MIN_SOLE_DEFAULT_CURVES).toBe(4); + }); + + it('returns the lone precision for a single-precision model regardless of count', () => { + expect(pickDefaultPrecisions({ fp4: 2 })).toEqual(['fp4']); + expect(pickDefaultPrecisions({ fp4: 10 })).toEqual(['fp4']); + }); + + it('returns [] when there are no precisions', () => { + expect(pickDefaultPrecisions({})).toEqual([]); + }); +}); + +describe('resolveEffectivePrecisions', () => { + const M3_COUNTS = { fp4: 1, fp8: 14 }; + const M3_AVAIL = ['fp4', 'fp8']; + + it('auto-defaults to both when one precision is sparse (M3 → fp4 + fp8)', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: M3_AVAIL, + curveCounts: M3_COUNTS, + explicit: false, + }), + ).toEqual(['fp4', 'fp8']); + }); + + it('auto-defaults to the densest precision when every precision is dense (dsr1 → fp8)', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: ['fp4', 'fp8'], + curveCounts: { fp4: 23, fp8: 38 }, + explicit: false, + }), + ).toEqual(['fp8']); + }); + + it('leaves an unchanged model on fp4 when fp4 is densest and both are dense', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: ['fp4', 'fp8'], + curveCounts: { fp4: 28, fp8: 5 }, + explicit: false, + }), + ).toEqual(['fp4']); + }); + + it('honors an explicit selection even when it is the sparse precision', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: M3_AVAIL, + curveCounts: M3_COUNTS, + explicit: true, + }), + ).toEqual(['fp4']); + }); + + it('honors an explicit multi-precision selection (e.g. a preset)', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4', 'fp8'], + availablePrecisions: M3_AVAIL, + curveCounts: M3_COUNTS, + explicit: true, + }), + ).toEqual(['fp4', 'fp8']); + }); + + it('drops explicitly-selected precisions that are unavailable, falling back to first available', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp8'], + availablePrecisions: ['fp4'], + curveCounts: { fp4: 10 }, + explicit: true, + }), + ).toEqual(['fp4']); + }); + + it('includes a loaded unofficial run precision so the overlay is visible by default', () => { + // Both official precisions dense → base is the sole densest (fp8); the user + // opened an fp4 unofficial run, so fp4 must be merged in. + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: ['fp4', 'fp8'], + curveCounts: { fp4: 23, fp8: 38 }, + unofficialPrecisions: ['fp4'], + explicit: false, + }), + ).toEqual(['fp4', 'fp8']); + }); + + it('ignores unofficial precisions that have no available data', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: ['fp4', 'fp8'], + curveCounts: { fp4: 23, fp8: 38 }, + unofficialPrecisions: ['int4'], + explicit: false, + }), + ).toEqual(['fp8']); + }); + + it('falls back to the first available precision when curve data is missing (still loading)', () => { + expect( + resolveEffectivePrecisions({ + selectedPrecisions: ['fp4'], + availablePrecisions: ['fp4'], + curveCounts: {}, + explicit: false, + }), + ).toEqual(['fp4']); + }); +}); diff --git a/packages/app/src/lib/default-precisions.ts b/packages/app/src/lib/default-precisions.ts new file mode 100644 index 00000000..71c49386 --- /dev/null +++ b/packages/app/src/lib/default-precisions.ts @@ -0,0 +1,118 @@ +import { Precision, PRECISION_OPTIONS } from './data-mappings'; + +/** + * Default precision selection. + * + * Historically every model defaulted to FP4-only, which left FP8-heavy models + * (e.g. MiniMax M3, whose FP4 lives on a single GPU while FP8 spans seven) + * showing a near-empty chart on first load. Instead we default to whichever + * precision has the most data, with a guard against committing to a sparse one. + * + * "Curves" = distinct (hardware, framework, spec_method, disagg) series that + * would render for a precision — i.e. how many lines the chart draws. + */ + +/** + * Curve threshold for picking a sole default. We only commit to a single + * precision when *every* available precision clears it; if any is below (e.g. + * MiniMax M3's lone FP4 curve next to its FP8 fleet) we show all of them so the + * sparse precision isn't silently dropped (per #470 discussion). + */ +export const MIN_SOLE_DEFAULT_CURVES = 4; + +interface CurveRow { + precision: string; + hardware: string; + framework: string; + spec_method: string; + disagg: boolean; +} + +/** Count distinct curves per precision from already-filtered rows (model + sequence). */ +export function countCurvesByPrecision(rows: CurveRow[]): Record { + const seen = new Map>(); + for (const r of rows) { + let curves = seen.get(r.precision); + if (!curves) { + curves = new Set(); + seen.set(r.precision, curves); + } + curves.add(`${r.hardware}|${r.framework}|${r.spec_method}|${r.disagg}`); + } + const counts: Record = {}; + for (const [precision, curves] of seen) counts[precision] = curves.size; + return counts; +} + +/** + * Tie-break rank (lower sorts first): FP4 wins ties so models that previously + * defaulted to FP4 are unchanged, then canonical enum order, unknowns last. + */ +function precisionRank(p: string): number { + if (p === Precision.FP4) return -1; + const i = (PRECISION_OPTIONS as readonly string[]).indexOf(p); + return i === -1 ? PRECISION_OPTIONS.length : i; +} + +function byRank(a: string, b: string): number { + return precisionRank(a) - precisionRank(b); +} + +/** + * Pick the default precision selection from per-precision curve counts: + * - the single densest precision when *every* available precision has + * >= `minCurves` curves; + * - otherwise every precision present, so a sparse precision (e.g. M3's lone FP4 + * alongside its dense FP8) is shown rather than silently dropped. + * Returns [] when there are no precisions. + */ +export function pickDefaultPrecisions( + counts: Record, + minCurves = MIN_SOLE_DEFAULT_CURVES, +): string[] { + const precisions = Object.keys(counts); + if (precisions.length === 0) return []; + const someSparse = precisions.some((p) => counts[p] < minCurves); + if (someSparse) return precisions.toSorted(byRank); + const densest = precisions.reduce((best, p) => { + if (counts[p] > counts[best]) return p; + if (counts[p] === counts[best] && precisionRank(p) < precisionRank(best)) return p; + return best; + }); + return [densest]; +} + +/** + * Resolve the effective precision selection for the current model + sequence. + * + * - When the user has explicitly chosen a precision (URL `i_prec`, a preset, or + * a manual toggle), honor it — intersected with what's available, falling back + * to the first available precision (preserves prior behavior). + * - Otherwise auto-pick the densest precision (see `pickDefaultPrecisions`) and + * union in any precisions present in a loaded unofficial run, so an overlay the + * user explicitly opened is visible by default instead of hidden behind FP4. + */ +export function resolveEffectivePrecisions(opts: { + selectedPrecisions: string[]; + availablePrecisions: string[]; + curveCounts: Record; + unofficialPrecisions?: string[]; + explicit: boolean; + minCurves?: number; +}): string[] { + const { selectedPrecisions, availablePrecisions, curveCounts, explicit } = opts; + const available = new Set(availablePrecisions); + + if (explicit) { + const valid = selectedPrecisions.filter((p) => available.has(p)); + if (valid.length > 0) return valid; + return availablePrecisions.length > 0 ? [availablePrecisions[0]] : selectedPrecisions; + } + + const base = pickDefaultPrecisions(curveCounts, opts.minCurves); + const merged = [...new Set([...base, ...(opts.unofficialPrecisions ?? [])])] + .filter((p) => available.has(p)) + .toSorted(byRank); + if (merged.length > 0) return merged; + return availablePrecisions.length > 0 ? [availablePrecisions[0]] : selectedPrecisions; +} diff --git a/packages/app/src/lib/url-state.ts b/packages/app/src/lib/url-state.ts index a5ba51d7..aa7481ed 100644 --- a/packages/app/src/lib/url-state.ts +++ b/packages/app/src/lib/url-state.ts @@ -64,7 +64,10 @@ export const PARAM_DEFAULTS: Record = { g_rundate: '', g_runid: '', i_seq: '8k/1k', - i_prec: 'fp4', + // No strippable default: precision is only written to the URL once chosen + // explicitly, so an explicit FP4 selection must survive (not be stripped as a + // "default") or it would silently revert to the per-model auto default on reload. + i_prec: '', i_metric: 'y_tpPerGpu', i_xmetric: 'p99_ttft', i_e2e_xmetric: '',