From 0a4c2b792f8828e1f9123670b9c63c9b6502cf52 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:18:37 -0500 Subject: [PATCH 1/3] feat(inference): known-issue warning boxes for affected configs --- .../components/inference/ui/ChartDisplay.tsx | 9 +- .../components/inference/ui/ScatterGraph.tsx | 82 ++++++++ .../utils/knownIssueAnnotations.test.ts | 148 +++++++++++++ .../inference/utils/knownIssueAnnotations.ts | 197 ++++++++++++++++++ packages/app/src/lib/csv-export.test.ts | 12 ++ packages/app/src/lib/csv-export.ts | 7 +- packages/app/src/lib/known-issues.test.ts | 70 +++++++ packages/app/src/lib/known-issues.ts | 77 +++++++ 8 files changed, 599 insertions(+), 3 deletions(-) create mode 100644 packages/app/src/components/inference/utils/knownIssueAnnotations.test.ts create mode 100644 packages/app/src/components/inference/utils/knownIssueAnnotations.ts create mode 100644 packages/app/src/lib/known-issues.test.ts create mode 100644 packages/app/src/lib/known-issues.ts diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 7671a566..145847f4 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -23,6 +23,8 @@ import { ChartShareActions, MetricAssumptionNotes } from '@/components/ui/chart- import { UnofficialDomainNotice } from '@/components/ui/unofficial-domain-notice'; import { exportToCsv } from '@/lib/csv-export'; import { inferenceChartToCsv } from '@/lib/csv-export-helpers'; +import { knownIssueCsvNote, matchKnownConfigIssues } from '@/lib/known-issues'; +import { getDisplayLabel } from '@/lib/utils'; import { Dialog, DialogContent, @@ -43,7 +45,7 @@ import { } from '@/lib/data-mappings'; import { useComparisonChangelogs } from '@/hooks/api/use-comparison-changelogs'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; -import { hardwareKeyMatchesAnyBase } from '@/lib/constants'; +import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; import ChartControls from './ChartControls'; import ComparisonChangelog from './ComparisonChangelog'; @@ -383,10 +385,15 @@ export default function ChartDisplay() { graph.model, graph.sequence, ); + const issueNotes = matchKnownConfigIssues(graph.model, visibleData).map( + (issue) => + knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), + ); exportToCsv( `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, headers, rows, + issueNotes, ); }} /> diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 56e0088e..61b4397f 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -11,6 +11,7 @@ import { useUnofficialRun } from '@/components/unofficial-run-provider'; import { computeToggle } from '@/hooks/useTogglableSet'; import { getHardwareConfig, getModelSortIndex } from '@/lib/constants'; import { getChartWatermark, getPrecisionLabel, type Precision } from '@/lib/data-mappings'; +import { matchKnownConfigIssues } from '@/lib/known-issues'; import { formatNumber, getDisplayLabel, updateRepoUrl } from '@/lib/utils'; import { D3Chart } from '@/lib/d3-chart/D3Chart'; import type { @@ -62,6 +63,10 @@ import { PARETO_LABEL_COLORS, buildGradientColorMap, } from '@/components/inference/utils/paretoLabels'; +import { + type KnownIssueAnnotation, + renderKnownIssueAnnotations, +} from '@/components/inference/utils/knownIssueAnnotations'; // X-shape path for overlay (unofficial) data points const X_SIZE = 5; @@ -112,6 +117,7 @@ const lineLabelText = (hwKey: string, precision: string, includePrecision: boole const ScatterGraph = React.memo( ({ chartId, + modelLabel, data, xLabel, yLabel, @@ -340,6 +346,35 @@ const ScatterGraph = React.memo( return overlayData.data.filter((p) => selectedPrecisions.includes(p.precision)); }, [overlayData, selectedPrecisions]); + // Warning annotations for visible series (official + unofficial overlay) + // with known upstream issues. Drawn as an SVG layer (box + arrow to the + // affected line) so PNG exports carry the warning. + const knownIssueAnnotations = useMemo((): KnownIssueAnnotation[] => { + const visibleOverlayPoints = processedOverlayData.filter((p) => + activeOverlayHwTypes.has(p.hwKey as string), + ); + const visiblePoints = [...filteredData, ...visibleOverlayPoints]; + return matchKnownConfigIssues(modelLabel, visiblePoints).map((issue) => ({ + issue, + label: parseHwKeyToLabel(issue.hwKey).label, + color: getCssColor(resolveColor(issue.hwKey)), + points: visiblePoints + .filter( + (p) => + String(p.hwKey) === issue.hwKey && + (!issue.precisions || issue.precisions.includes(p.precision)), + ) + .map((p) => ({ x: p.x, y: p.y })), + })); + }, [ + modelLabel, + filteredData, + processedOverlayData, + activeOverlayHwTypes, + resolveColor, + getCssColor, + ]); + // Combined data for D3 scale domain (includes overlay so scales fit both datasets) const chartScaleData = useMemo(() => { if (processedOverlayData.length === 0) return filteredData; @@ -1760,11 +1795,58 @@ const ScatterGraph = React.memo( }, }; + // ── Known-issue annotations: warning box + arrow to the affected line ── + // Boxes right-align against the legend panel, which floats over the + // SVG's right edge when expanded — measure the live overlap so they sit + // beside it rather than under it. + const knownIssueRightInset = (ctx: RenderContext): number => { + const svgNode = ctx.layout.svg.node(); + const legend = document.querySelector(`#${chartId} .legend-container`); + if (!svgNode || !legend) return 0; + const innerRight = + svgNode.getBoundingClientRect().left + ctx.layout.margin.left + ctx.width; + const overlap = innerRight - legend.getBoundingClientRect().left; + return Math.max(0, Math.min(overlap, ctx.width * 0.4)); + }; + const drawKnownIssues = ( + ctx: RenderContext, + xScale: ContinuousScale, + yScale: ContinuousScale, + ) => { + renderKnownIssueAnnotations(ctx.layout.g, ctx.layout.defs, { + chartId, + width: ctx.width, + height: ctx.height, + xScale, + yScale, + annotations: knownIssueAnnotations, + rightInset: knownIssueRightInset(ctx), + background: getCssColor('--background'), + foreground: getCssColor('--foreground'), + mutedForeground: getCssColor('--muted-foreground'), + onLinkClick: (a) => + track('inference_known_issue_clicked', { + hwKey: a.issue.hwKey, + issue: a.issue.issueRef, + }), + }); + }; + const knownIssueLayer: CustomLayerConfig = { + type: 'custom', + key: 'known-issues', + render: (_zoomGroup, ctx) => + drawKnownIssues(ctx, ctx.xScale as ContinuousScale, ctx.yScale as ContinuousScale), + onZoom: (_zoomGroup, ctx) => + drawKnownIssues(ctx, ctx.newXScale as ContinuousScale, ctx.newYScale as ContinuousScale), + }; + const result: LayerConfig[] = [rooflineLayer, scatterLayer]; if (overlayLayer) result.push(overlayLayer); result.push(speedOverlayLayer); + result.push(knownIssueLayer); return result; }, [ + knownIssueAnnotations, rooflines, allPointLabelsByKey, showGradientLabels, diff --git a/packages/app/src/components/inference/utils/knownIssueAnnotations.test.ts b/packages/app/src/components/inference/utils/knownIssueAnnotations.test.ts new file mode 100644 index 00000000..5b385dc6 --- /dev/null +++ b/packages/app/src/components/inference/utils/knownIssueAnnotations.test.ts @@ -0,0 +1,148 @@ +// @vitest-environment jsdom +import * as d3 from 'd3'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { KNOWN_CONFIG_ISSUES } from '@/lib/known-issues'; + +import { + type AnnotationRenderOptions, + type KnownIssueAnnotation, + renderKnownIssueAnnotations, +} from './knownIssueAnnotations'; + +let g: d3.Selection; +let defs: d3.Selection; + +const gb300Annotation: KnownIssueAnnotation = { + issue: KNOWN_CONFIG_ISSUES[0], + label: 'GB300 NVL72 (Dynamo TRT, MTP)', + color: 'rgb(118, 185, 0)', + points: [ + { x: 100, y: 300 }, + { x: 200, y: 200 }, + ], +}; + +const mi355xAnnotation: KnownIssueAnnotation = { + issue: KNOWN_CONFIG_ISSUES[1], + label: 'MI355X (MoRI SGLang, MTP)', + color: 'rgb(237, 28, 36)', + points: [{ x: 150, y: 250 }], +}; + +function baseOptions(overrides: Partial = {}): AnnotationRenderOptions { + return { + chartId: 'chart-0', + width: 800, + height: 500, + xScale: (v) => v, + yScale: (v) => v, + annotations: [gb300Annotation, mi355xAnnotation], + background: '#fff', + foreground: '#111', + mutedForeground: '#666', + ...overrides, + }; +} + +beforeEach(() => { + const svg = d3 + .select(document.body) + .html('') + .append('svg') + .attr('width', 800) + .attr('height', 500); + defs = svg.append('defs'); + g = svg.append('g'); +}); + +describe('renderKnownIssueAnnotations', () => { + it('renders nothing when there are no annotations', () => { + renderKnownIssueAnnotations(g, defs, baseOptions({ annotations: [] })); + expect(g.select('.known-issue-annotations').empty()).toBe(true); + }); + + it('renders one linked warning box per annotation with label and issue ref', () => { + renderKnownIssueAnnotations(g, defs, baseOptions()); + + const boxes = g.selectAll('[data-testid="known-issue-annotation"]').nodes() as SVGAElement[]; + expect(boxes).toHaveLength(2); + + expect(boxes[0].getAttribute('href')).toBe('https://github.com/NVIDIA/srt-slurm/issues/51'); + expect(boxes[0].getAttribute('target')).toBe('_blank'); + expect(boxes[0].textContent).toContain('GB300 NVL72 (Dynamo TRT, MTP)'); + expect(boxes[0].textContent).toContain('Accuracy issues — filed since Apr 21, 2026'); + expect(boxes[0].textContent).toContain('NVIDIA/srt-slurm#51'); + + expect(boxes[1].getAttribute('href')).toBe( + 'https://github.com/sgl-project/sglang/issues/27194', + ); + expect(boxes[1].textContent).toContain('MI355X (MoRI SGLang, MTP)'); + }); + + it('stacks boxes without overlap, right-aligned inside the plot', () => { + renderKnownIssueAnnotations(g, defs, baseOptions()); + + const rects = g.selectAll('.known-issue-annotation rect').nodes() as SVGRectElement[]; + expect(rects).toHaveLength(2); + const top0 = Number(rects[0].getAttribute('y')); + const bottom0 = top0 + Number(rects[0].getAttribute('height')); + const top1 = Number(rects[1].getAttribute('y')); + expect(top1).toBeGreaterThan(bottom0); + for (const rect of rects) { + expect(Number(rect.getAttribute('x'))).toBeGreaterThanOrEqual(0); + // Right edges align at the plot's right edge minus the gap + expect(Number(rect.getAttribute('x')) + Number(rect.getAttribute('width'))).toBe(800 - 10); + } + }); + + it('shifts boxes left of a floating legend via rightInset', () => { + renderKnownIssueAnnotations(g, defs, baseOptions({ rightInset: 120 })); + + const rects = g.selectAll('.known-issue-annotation rect').nodes() as SVGRectElement[]; + for (const rect of rects) { + expect(Number(rect.getAttribute('x')) + Number(rect.getAttribute('width'))).toBe( + 800 - 10 - 120, + ); + } + }); + + it('draws a series-colored arrow with an arrowhead marker per annotation', () => { + renderKnownIssueAnnotations(g, defs, baseOptions()); + + const arrows = g.selectAll('.known-issue-arrow').nodes() as SVGPathElement[]; + expect(arrows).toHaveLength(2); + expect(arrows[0].getAttribute('stroke')).toBe('rgb(118, 185, 0)'); + expect(arrows[0].getAttribute('marker-end')).toBe('url(#known-issue-arrowhead-chart-0-0)'); + expect(defs.select('#known-issue-arrowhead-chart-0-1').empty()).toBe(false); + }); + + it('omits the arrow when the series is zoomed/panned out of view', () => { + // Scales push every point far outside the 800x500 plot + renderKnownIssueAnnotations( + g, + defs, + baseOptions({ annotations: [gb300Annotation], xScale: (v) => v + 10_000 }), + ); + + expect(g.selectAll('[data-testid="known-issue-annotation"]').nodes()).toHaveLength(1); + expect(g.selectAll('.known-issue-arrow').nodes()).toHaveLength(0); + }); + + it('re-rendering replaces the previous annotations instead of accumulating', () => { + renderKnownIssueAnnotations(g, defs, baseOptions()); + renderKnownIssueAnnotations(g, defs, baseOptions({ annotations: [gb300Annotation] })); + + expect(g.selectAll('[data-testid="known-issue-annotation"]').nodes()).toHaveLength(1); + expect(defs.selectAll('marker').nodes()).toHaveLength(1); + }); + + it('fires the click callback with the annotation', () => { + const onLinkClick = vi.fn(); + renderKnownIssueAnnotations(g, defs, baseOptions({ onLinkClick })); + + const box = g.select('[data-testid="known-issue-annotation"]').node() as SVGAElement; + box.dispatchEvent(new MouseEvent('click', { bubbles: true })); + expect(onLinkClick).toHaveBeenCalledWith(gb300Annotation); + }); +}); diff --git a/packages/app/src/components/inference/utils/knownIssueAnnotations.ts b/packages/app/src/components/inference/utils/knownIssueAnnotations.ts new file mode 100644 index 00000000..5370c1f6 --- /dev/null +++ b/packages/app/src/components/inference/utils/knownIssueAnnotations.ts @@ -0,0 +1,197 @@ +/** + * SVG renderer for known-issue warning annotations on the inference scatter + * chart: a stacked column of warning boxes at the top of the plot, each with + * an arrow pointing at its affected series. Drawn inside the chart SVG so PNG + * exports carry the warnings; colors are passed in resolved because CSS + * variables don't survive html-to-image. + */ + +import type * as d3 from 'd3'; + +import type { KnownConfigIssue } from '@/lib/known-issues'; + +export interface KnownIssueAnnotation { + issue: KnownConfigIssue; + /** Display label of the affected series, e.g. "GB300 NVL72 (Dynamo TRT, MTP)" */ + label: string; + /** Resolved stroke color of the affected series */ + color: string; + /** Data-space coordinates of the series' visible points (arrow targets) */ + points: { x: number; y: number }[]; +} + +export interface AnnotationRenderOptions { + chartId: string; + width: number; + height: number; + xScale: (value: number) => number; + yScale: (value: number) => number; + annotations: KnownIssueAnnotation[]; + background: string; + foreground: string; + mutedForeground: string; + /** + * Horizontal space reserved at the plot's right edge (e.g. for a legend + * panel floating over the SVG). Boxes right-align against it. + */ + rightInset?: number; + onLinkClick?: (annotation: KnownIssueAnnotation) => void; +} + +const BOX_TOP = 8; +const BOX_GAP = 8; +const BOX_RIGHT_GAP = 10; +const PAD_X = 10; +const PAD_Y = 7; +const LINE1_SIZE = 11; +const LINE2_SIZE = 10; +const LINE1_H = 14; +const LINE2_H = 13; +const SWATCH_R = 4; +const SWATCH_SPACE = 14; +const ARROW_STANDOFF = 9; +const MIN_ARROW_LEN = 24; + +/** getComputedTextLength with an estimate fallback for jsdom/test environments. */ +function measureTextWidth(node: SVGTextElement | null, chars: number, fontSize: number): number { + if (node && typeof node.getComputedTextLength === 'function') { + try { + const len = node.getComputedTextLength(); + if (Number.isFinite(len) && len > 0) return len; + } catch { + // fall through to estimate + } + } + return chars * fontSize * 0.58; +} + +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +export function renderKnownIssueAnnotations( + g: d3.Selection, + defs: d3.Selection, + opts: AnnotationRenderOptions, +): void { + const { chartId, width, height, xScale, yScale, annotations } = opts; + + g.selectAll('.known-issue-annotations').remove(); + defs.selectAll(`[id^="known-issue-arrowhead-${chartId}"]`).remove(); + if (annotations.length === 0) return; + + const layer = g.append('g').attr('class', 'known-issue-annotations'); + const arrowGroup = layer.append('g').attr('class', 'known-issue-arrows'); + + let yCursor = BOX_TOP; + annotations.forEach((annotation, index) => { + const { issue, label, color, points } = annotation; + + const markerId = `known-issue-arrowhead-${chartId}-${index}`; + defs + .append('marker') + .attr('id', markerId) + .attr('viewBox', '0 0 10 10') + .attr('refX', 8) + .attr('refY', 5) + .attr('markerWidth', 7) + .attr('markerHeight', 7) + .attr('orient', 'auto') + .append('path') + .attr('d', 'M 0 0 L 10 5 L 0 10 z') + .attr('fill', color); + + const anchor = layer + .append('a') + .attr('href', issue.url) + .attr('target', '_blank') + .attr('rel', 'noopener noreferrer') + .attr('class', 'known-issue-annotation') + .attr('data-testid', 'known-issue-annotation') + .attr('cursor', 'pointer') + .on('click', () => opts.onLinkClick?.(annotation)); + + const detail = `${issue.summary} — filed since ${issue.filed} · `; + const text1 = anchor + .append('text') + .attr('font-size', LINE1_SIZE) + .attr('font-weight', 600) + .attr('fill', opts.foreground) + .text(label); + const text2 = anchor.append('text').attr('font-size', LINE2_SIZE); + text2.append('tspan').attr('fill', opts.mutedForeground).text(detail); + text2 + .append('tspan') + .attr('fill', opts.foreground) + .attr('text-decoration', 'underline') + .text(issue.issueRef); + + const w1 = measureTextWidth(text1.node(), label.length, LINE1_SIZE) + SWATCH_SPACE; + const w2 = measureTextWidth(text2.node(), detail.length + issue.issueRef.length, LINE2_SIZE); + const boxW = Math.max(w1, w2) + PAD_X * 2; + const boxH = PAD_Y * 2 + LINE1_H + LINE2_H; + const boxRight = width - BOX_RIGHT_GAP - (opts.rightInset ?? 0); + const bx = Math.max(2, boxRight - boxW); + const by = yCursor; + + anchor + .insert('rect', 'text') + .attr('x', bx) + .attr('y', by) + .attr('width', boxW) + .attr('height', boxH) + .attr('rx', 6) + .attr('fill', opts.background) + .attr('fill-opacity', 0.95) + .attr('stroke', '#f59e0b') + .attr('stroke-width', 1.5); + + anchor + .append('circle') + .attr('cx', bx + PAD_X + SWATCH_R) + .attr('cy', by + PAD_Y + LINE1_H / 2) + .attr('r', SWATCH_R) + .attr('fill', color); + text1.attr('x', bx + PAD_X + SWATCH_SPACE).attr('y', by + PAD_Y + LINE1_H - 4); + text2.attr('x', bx + PAD_X).attr('y', by + PAD_Y + LINE1_H + LINE2_H - 4); + + // Arrow from the box's bottom edge to the nearest on-screen point of the + // affected series. Skipped when the series is panned/zoomed out of view or + // the target sits right under the box. + const bottomY = by + boxH; + const onScreen = points + .map((p) => ({ px: xScale(p.x), py: yScale(p.y) })) + .filter((p) => p.px >= 0 && p.px <= width && p.py >= 0 && p.py <= height); + if (onScreen.length > 0) { + const target = onScreen.reduce((best, p) => { + const startX = clamp(p.px, bx + 12, bx + boxW - 12); + const bestStartX = clamp(best.px, bx + 12, bx + boxW - 12); + const dist = Math.hypot(p.px - startX, p.py - bottomY); + const bestDist = Math.hypot(best.px - bestStartX, best.py - bottomY); + return dist < bestDist ? p : best; + }); + const startX = clamp(target.px, bx + 12, bx + boxW - 12); + const dx = target.px - startX; + const dy = target.py - bottomY; + const len = Math.hypot(dx, dy); + if (len >= MIN_ARROW_LEN) { + const endX = target.px - (dx / len) * ARROW_STANDOFF; + const endY = target.py - (dy / len) * ARROW_STANDOFF; + // Slight quadratic bend so stacked arrows don't read as one straight rule + const midX = (startX + endX) / 2 + (dy > 0 ? -dx * 0.12 : dx * 0.12); + const midY = (bottomY + endY) / 2; + arrowGroup + .append('path') + .attr('class', 'known-issue-arrow') + .attr('d', `M ${startX} ${bottomY + 2} Q ${midX} ${midY} ${endX} ${endY}`) + .attr('fill', 'none') + .attr('stroke', color) + .attr('stroke-width', 1.75) + .attr('marker-end', `url(#${markerId})`) + .attr('pointer-events', 'none'); + } + } + + yCursor += boxH + BOX_GAP; + }); +} diff --git a/packages/app/src/lib/csv-export.test.ts b/packages/app/src/lib/csv-export.test.ts index 76b3d745..1f136faf 100644 --- a/packages/app/src/lib/csv-export.test.ts +++ b/packages/app/src/lib/csv-export.test.ts @@ -32,6 +32,18 @@ describe('buildCsv', () => { expect(lines[2]).toBe('B200 NVL72,88.1,false'); }); + it('inserts notes as comment lines between the preamble and the header', () => { + const note = 'WARNING: GB300 NVL72 (Dynamo TRT, MTP) — accuracy issues reported'; + const result = buildCsv(['A'], [['1']], [note]); + const lines = result.split('\n'); + + const noteIndex = lines.indexOf(`# ${note}`); + expect(noteIndex).toBeGreaterThan(-1); + expect(lines[noteIndex + 1]).toBe('A'); + // Notes must not disturb the data section + expect(dataLines(result)).toBe('A\n1'); + }); + it('escapes cells containing commas', () => { const result = buildCsv(['Label'], [['H100, SXM']]); expect(dataLines(result)).toBe('Label\n"H100, SXM"'); diff --git a/packages/app/src/lib/csv-export.ts b/packages/app/src/lib/csv-export.ts index e36d4de4..b3c17459 100644 --- a/packages/app/src/lib/csv-export.ts +++ b/packages/app/src/lib/csv-export.ts @@ -26,10 +26,12 @@ function escapeCsvCell(value: string | number | boolean | null | undefined): str export function buildCsv( headers: string[], rows: (string | number | boolean | null | undefined)[][], + notes: string[] = [], ): string { + const noteLines = notes.map((note) => `# ${note}`); const headerLine = headers.map(escapeCsvCell).join(','); const dataLines = rows.map((row) => row.map(escapeCsvCell).join(',')); - return [csvLicensePreamble(), headerLine, ...dataLines].join('\n'); + return [csvLicensePreamble(), ...noteLines, headerLine, ...dataLines].join('\n'); } /** Trigger a CSV file download in the browser */ @@ -48,7 +50,8 @@ export function exportToCsv( filename: string, headers: string[], rows: (string | number | boolean | null | undefined)[][], + notes: string[] = [], ): void { - const csv = buildCsv(headers, rows); + const csv = buildCsv(headers, rows, notes); downloadCsv(filename, csv); } diff --git a/packages/app/src/lib/known-issues.test.ts b/packages/app/src/lib/known-issues.test.ts new file mode 100644 index 00000000..4b92f786 --- /dev/null +++ b/packages/app/src/lib/known-issues.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; + +import { KNOWN_CONFIG_ISSUES, knownIssueCsvNote, matchKnownConfigIssues } from './known-issues'; + +const DSR1 = 'DeepSeek-R1-0528'; + +describe('matchKnownConfigIssues', () => { + it('matches the GB300 Dynamo TRT MTP entry for DeepSeek R1 FP4', () => { + const issues = matchKnownConfigIssues(DSR1, [ + { hwKey: 'gb300_dynamo-trt_mtp', precision: 'fp4' }, + ]); + expect(issues).toHaveLength(1); + expect(issues[0].url).toBe('https://github.com/NVIDIA/srt-slurm/issues/51'); + }); + + it('does not match GB300 Dynamo TRT MTP for non-FP4 precisions', () => { + const issues = matchKnownConfigIssues(DSR1, [ + { hwKey: 'gb300_dynamo-trt_mtp', precision: 'fp8' }, + ]); + expect(issues).toHaveLength(0); + }); + + it('matches the MI355X MoRI SGLang MTP entry regardless of precision', () => { + for (const precision of ['fp4', 'fp8']) { + const issues = matchKnownConfigIssues(DSR1, [{ hwKey: 'mi355x_mori-sglang_mtp', precision }]); + expect(issues).toHaveLength(1); + expect(issues[0].url).toBe('https://github.com/sgl-project/sglang/issues/27194'); + } + }); + + it('does not match other models', () => { + const issues = matchKnownConfigIssues('DeepSeek-V4-Pro', [ + { hwKey: 'gb300_dynamo-trt_mtp', precision: 'fp4' }, + { hwKey: 'mi355x_mori-sglang_mtp', precision: 'fp4' }, + ]); + expect(issues).toHaveLength(0); + }); + + it('does not match unaffected configs (non-MTP, other hardware)', () => { + const issues = matchKnownConfigIssues(DSR1, [ + { hwKey: 'gb300_dynamo-trt', precision: 'fp4' }, + { hwKey: 'mi355x_sglang', precision: 'fp4' }, + { hwKey: 'b200_trt_mtp', precision: 'fp4' }, + ]); + expect(issues).toHaveLength(0); + }); + + it('returns each issue at most once even with many matching points', () => { + const issues = matchKnownConfigIssues(DSR1, [ + { hwKey: 'gb300_dynamo-trt_mtp', precision: 'fp4' }, + { hwKey: 'gb300_dynamo-trt_mtp', precision: 'fp4' }, + { hwKey: 'mi355x_mori-sglang_mtp', precision: 'fp4' }, + ]); + expect(issues).toHaveLength(2); + }); + + it('returns nothing for an empty point list', () => { + expect(matchKnownConfigIssues(DSR1, [])).toHaveLength(0); + }); +}); + +describe('knownIssueCsvNote', () => { + it('includes the config label, filing date, issue ref, and URL', () => { + const note = knownIssueCsvNote(KNOWN_CONFIG_ISSUES[0], 'GB300 NVL72 (Dynamo TRT, MTP)'); + expect(note).toContain('WARNING: GB300 NVL72 (Dynamo TRT, MTP)'); + expect(note).toContain('filed since Apr 21, 2026'); + expect(note).toContain('NVIDIA/srt-slurm#51'); + expect(note).toContain('https://github.com/NVIDIA/srt-slurm/issues/51'); + }); +}); diff --git a/packages/app/src/lib/known-issues.ts b/packages/app/src/lib/known-issues.ts new file mode 100644 index 00000000..828216ca --- /dev/null +++ b/packages/app/src/lib/known-issues.ts @@ -0,0 +1,77 @@ +/** + * Known upstream issues affecting specific benchmark configurations. + * + * Entries are matched against the series visible on an inference chart. + * Matches render an on-chart warning box (which is captured by PNG export) + * and are stamped into the CSV export header. + */ + +import { Model } from '@/lib/data-mappings'; + +export interface KnownConfigIssue { + /** Availability hardware key the issue applies to ({hardware}_{framework}[_mtp]) */ + hwKey: string; + /** Model the issue applies to */ + model: Model; + /** Precisions the issue applies to; omit to match every precision */ + precisions?: string[]; + /** Short description shown in the warning box, e.g. "Accuracy issues" */ + summary: string; + /** Human-readable filing date, e.g. "Apr 21, 2026" */ + filed: string; + /** Upstream issue URL */ + url: string; + /** Short issue reference, e.g. "NVIDIA/srt-slurm#51" */ + issueRef: string; +} + +export const KNOWN_CONFIG_ISSUES: KnownConfigIssue[] = [ + { + hwKey: 'gb300_dynamo-trt_mtp', + model: Model.DeepSeek_R1, + precisions: ['fp4'], + summary: 'Accuracy issues', + filed: 'Apr 21, 2026', + url: 'https://github.com/NVIDIA/srt-slurm/issues/51', + issueRef: 'NVIDIA/srt-slurm#51', + }, + { + hwKey: 'mi355x_mori-sglang_mtp', + model: Model.DeepSeek_R1, + summary: 'Accuracy issues', + filed: 'Jun 4, 2026', + url: 'https://github.com/sgl-project/sglang/issues/27194', + issueRef: 'sgl-project/sglang#27194', + }, +]; + +/** Minimal point shape needed for matching. */ +export interface MatchablePoint { + hwKey: string | number; + precision: string; +} + +/** + * Return the known issues whose (model, hwKey, precision) matches at least one + * visible chart point. Order follows KNOWN_CONFIG_ISSUES; each issue appears at + * most once. + */ +export function matchKnownConfigIssues( + model: string, + points: MatchablePoint[], +): KnownConfigIssue[] { + return KNOWN_CONFIG_ISSUES.filter( + (issue) => + issue.model === model && + points.some( + (p) => + String(p.hwKey) === issue.hwKey && + (!issue.precisions || issue.precisions.includes(p.precision)), + ), + ); +} + +/** Format a known issue as a CSV header comment line. */ +export function knownIssueCsvNote(issue: KnownConfigIssue, configLabel: string): string { + return `WARNING: ${configLabel} — ${issue.summary.toLowerCase()} reported, filed since ${issue.filed} (${issue.issueRef}): ${issue.url}`; +} From 2902f7e1dba7e7988fb83da4c61df17d7995a5a6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:41:17 -0500 Subject: [PATCH 2/3] fix(inference): include overlay series in CSV export issue warnings --- .../components/inference/ui/ChartDisplay.tsx | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 145847f4..47dfe600 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -173,8 +173,14 @@ export default function ChartDisplay() { track('inference_view_changed', { view: value, chartIndex: index }); }; - const { unofficialRunInfo, unofficialRunInfos, runIndexByUrl, getOverlayData, isUnofficialRun } = - useUnofficialRun(); + const { + unofficialRunInfo, + unofficialRunInfos, + runIndexByUrl, + getOverlayData, + isUnofficialRun, + activeOverlayHwTypes, + } = useUnofficialRun(); // Compute overlay data for each chart type — must match useChartData processing const overlayDataByChartType = useMemo(() => { @@ -385,9 +391,24 @@ export default function ChartDisplay() { graph.model, graph.sequence, ); - const issueNotes = matchKnownConfigIssues(graph.model, visibleData).map( - (issue) => - knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), + // Match warnings against the same series the chart annotates, + // including visible unofficial-run overlay series. + const overlay = + graph.chartDefinition.chartType === 'e2e' + ? overlayDataByChartType.e2e + : overlayDataByChartType.interactivity; + const visibleOverlayRows = isTimelineMode + ? [] + : (overlay?.data ?? []).filter( + (p) => + activeOverlayHwTypes.has(p.hwKey as string) && + selectedPrecisions.includes(p.precision), + ); + const issueNotes = matchKnownConfigIssues(graph.model, [ + ...visibleData, + ...visibleOverlayRows, + ]).map((issue) => + knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), ); exportToCsv( `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, From 2e0e554af21c02c6fc50134005ae44008753bf01 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:41:24 -0500 Subject: [PATCH 3/3] feat(inference): show known-issue warnings on GPU date-comparison view --- .../src/components/inference/ui/GPUGraph.tsx | 73 ++++++++++++++++++- .../components/inference/ui/ScatterGraph.tsx | 20 ++--- .../inference/utils/knownIssueAnnotations.ts | 19 +++++ 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/packages/app/src/components/inference/ui/GPUGraph.tsx b/packages/app/src/components/inference/ui/GPUGraph.tsx index dd860fc7..a6e76fcc 100644 --- a/packages/app/src/components/inference/ui/GPUGraph.tsx +++ b/packages/app/src/components/inference/ui/GPUGraph.tsx @@ -42,6 +42,12 @@ import { generateGPUGraphTooltipContent, getPointLabel, } from '@/components/inference/utils/tooltipUtils'; +import { + type KnownIssueAnnotation, + measureLegendRightInset, + renderKnownIssueAnnotations, +} from '@/components/inference/utils/knownIssueAnnotations'; +import { matchKnownConfigIssues } from '@/lib/known-issues'; const CHART_MARGIN = { top: 24, right: 10, bottom: 60, left: 60 }; @@ -58,7 +64,7 @@ function labelTextFor(pts: InferenceData[]): string { } const GPUGraph = React.memo( - ({ chartId, data, xLabel, yLabel, chartDefinition, caption }: ScatterGraphProps) => { + ({ chartId, modelLabel, data, xLabel, yLabel, chartDefinition, caption }: ScatterGraphProps) => { const { hardwareConfig, selectedPrecisions, @@ -211,6 +217,70 @@ const GPUGraph = React.memo( return pts; }, [groupedData, activeDates, hideNonOptimal, optimalPointKeys]); + // Warning annotations for visible series with known upstream issues — + // same treatment the scatter view gets, applied to the date-comparison view. + // Lines here are colored per (gpu, date) pair, so take the first active + // pair's color as the series swatch. + const knownIssueAnnotations = useMemo( + (): KnownIssueAnnotation[] => + matchKnownConfigIssues(modelLabel, filteredData).map((issue) => { + const cfg = getHardwareConfig(issue.hwKey); + const colorEntry = allGraphs.find( + (entry) => entry.hwKey === issue.hwKey && activeDates.has(entry.id), + ); + return { + issue, + label: cfg ? getDisplayLabel(cfg) : issue.hwKey, + color: getCssColor(colorEntry?.color ?? resolveColor(issue.hwKey)), + points: filteredData + .filter( + (p) => + String(p.hwKey) === issue.hwKey && + (!issue.precisions || issue.precisions.includes(p.precision)), + ) + .map((p) => ({ x: p.x, y: p.y })), + }; + }), + [modelLabel, filteredData, allGraphs, activeDates, resolveColor, getCssColor], + ); + + const drawKnownIssues = ( + ctx: RenderContext, + xScale: ContinuousScale, + yScale: ContinuousScale, + ) => { + renderKnownIssueAnnotations(ctx.layout.g, ctx.layout.defs, { + chartId, + width: ctx.width, + height: ctx.height, + xScale, + yScale, + annotations: knownIssueAnnotations, + rightInset: measureLegendRightInset( + chartId, + ctx.layout.svg.node(), + ctx.layout.margin.left, + ctx.width, + ), + background: getCssColor('--background'), + foreground: getCssColor('--foreground'), + mutedForeground: getCssColor('--muted-foreground'), + onLinkClick: (a) => + track('inference_known_issue_clicked', { + hwKey: a.issue.hwKey, + issue: a.issue.issueRef, + }), + }); + }; + const knownIssueLayer: CustomLayerConfig = { + type: 'custom', + key: 'known-issues', + render: (_zoomGroup, ctx) => + drawKnownIssues(ctx, ctx.xScale as ContinuousScale, ctx.yScale as ContinuousScale), + onZoom: (_zoomGroup, ctx) => + drawKnownIssues(ctx, ctx.newXScale as ContinuousScale, ctx.newYScale as ContinuousScale), + }; + // Compute scale domains const xExtent = useMemo(() => { if (filteredData.length === 0) return [0, 100] as [number, number]; @@ -649,6 +719,7 @@ const GPUGraph = React.memo( }, }, lineLabelLayer, + knownIssueLayer, ]} zoom={{ enabled: true, diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 61b4397f..fa383ada 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -65,6 +65,7 @@ import { } from '@/components/inference/utils/paretoLabels'; import { type KnownIssueAnnotation, + measureLegendRightInset, renderKnownIssueAnnotations, } from '@/components/inference/utils/knownIssueAnnotations'; @@ -1796,18 +1797,6 @@ const ScatterGraph = React.memo( }; // ── Known-issue annotations: warning box + arrow to the affected line ── - // Boxes right-align against the legend panel, which floats over the - // SVG's right edge when expanded — measure the live overlap so they sit - // beside it rather than under it. - const knownIssueRightInset = (ctx: RenderContext): number => { - const svgNode = ctx.layout.svg.node(); - const legend = document.querySelector(`#${chartId} .legend-container`); - if (!svgNode || !legend) return 0; - const innerRight = - svgNode.getBoundingClientRect().left + ctx.layout.margin.left + ctx.width; - const overlap = innerRight - legend.getBoundingClientRect().left; - return Math.max(0, Math.min(overlap, ctx.width * 0.4)); - }; const drawKnownIssues = ( ctx: RenderContext, xScale: ContinuousScale, @@ -1820,7 +1809,12 @@ const ScatterGraph = React.memo( xScale, yScale, annotations: knownIssueAnnotations, - rightInset: knownIssueRightInset(ctx), + rightInset: measureLegendRightInset( + chartId, + ctx.layout.svg.node(), + ctx.layout.margin.left, + ctx.width, + ), background: getCssColor('--background'), foreground: getCssColor('--foreground'), mutedForeground: getCssColor('--muted-foreground'), diff --git a/packages/app/src/components/inference/utils/knownIssueAnnotations.ts b/packages/app/src/components/inference/utils/knownIssueAnnotations.ts index 5370c1f6..91748102 100644 --- a/packages/app/src/components/inference/utils/knownIssueAnnotations.ts +++ b/packages/app/src/components/inference/utils/knownIssueAnnotations.ts @@ -69,6 +69,25 @@ function clamp(value: number, min: number, max: number): number { return Math.max(min, Math.min(max, value)); } +/** + * Live overlap of the floating legend panel over the SVG's right edge, used + * as `rightInset` so warning boxes sit beside the legend instead of under it. + * Returns 0 when no legend is present (or outside the browser). + */ +export function measureLegendRightInset( + chartId: string, + svgNode: SVGSVGElement | null, + marginLeft: number, + width: number, +): number { + if (!svgNode || typeof document === 'undefined') return 0; + const legend = document.querySelector(`#${chartId} .legend-container`); + if (!legend) return 0; + const innerRight = svgNode.getBoundingClientRect().left + marginLeft + width; + const overlap = innerRight - legend.getBoundingClientRect().left; + return clamp(overlap, 0, width * 0.4); +} + export function renderKnownIssueAnnotations( g: d3.Selection, defs: d3.Selection,