From 0fff2b54ba50c7bfc63ac869e59124364dffa43f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 21 Jul 2026 15:00:17 -0500 Subject: [PATCH 1/3] show AgentX observed-window variability --- .../cypress/component/scatter-graph.cy.tsx | 10 + .../app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 89 +++++ .../inference/hooks/useChartData.test.ts | 25 ++ .../inference/hooks/useChartData.ts | 43 ++- .../app/src/components/inference/types.ts | 31 ++ .../ui/ScatterGraph.decoration.test.tsx | 110 ++++++- .../components/inference/ui/ScatterGraph.tsx | 305 +++++++++++++++++- .../src/components/inference/utils.test.ts | 18 ++ .../app/src/components/inference/utils.ts | 3 +- .../utils/observed-window-range.test.ts | 90 ++++++ .../inference/utils/observed-window-range.ts | 41 +++ .../inference/utils/tooltip-utils.test.ts | 56 ++++ .../inference/utils/tooltipUtils.ts | 103 ++++++ .../app/src/lib/benchmark-transform.test.ts | 33 ++ packages/app/src/lib/benchmark-transform.ts | 19 ++ packages/constants/src/metric-keys.ts | 22 ++ .../db/research/agentx-paired-signatures.sql | 65 ++++ .../db/research/agentx-trace-coverage.sql | 113 +++++++ .../agentx-uncertainty-diagnostics.sql | 228 +++++++++++++ .../db/research/agentx-window-sensitivity.sql | 127 ++++++++ packages/db/src/etl/agentic-v3-flatten.ts | 35 ++ packages/db/src/etl/benchmark-mapper.test.ts | 33 ++ 22 files changed, 1577 insertions(+), 22 deletions(-) create mode 100644 packages/app/src/components/inference/utils/observed-window-range.test.ts create mode 100644 packages/app/src/components/inference/utils/observed-window-range.ts create mode 100644 packages/db/research/agentx-paired-signatures.sql create mode 100644 packages/db/research/agentx-trace-coverage.sql create mode 100644 packages/db/research/agentx-uncertainty-diagnostics.sql create mode 100644 packages/db/research/agentx-window-sensitivity.sql diff --git a/packages/app/cypress/component/scatter-graph.cy.tsx b/packages/app/cypress/component/scatter-graph.cy.tsx index 692f6a77..dba15bfc 100644 --- a/packages/app/cypress/component/scatter-graph.cy.tsx +++ b/packages/app/cypress/component/scatter-graph.cy.tsx @@ -732,6 +732,10 @@ describe('ScatterGraph', () => { y: 260 - index * 40, precision: Precision.FP4, run_url: runUrl, + observedXMin: x * 0.75, + observedXMax: x * 1.25, + observed_window_seconds: 600, + observed_window_count: 6, }), ), hardwareConfig: hwConfig, @@ -777,10 +781,16 @@ describe('ScatterGraph', () => { cy.get('#test-scatter-dismiss-preview svg .unofficial-overlay-pt').should('have.length', 3); cy.get('#test-scatter-dismiss-preview svg .overlay-roofline-path').should('exist'); + cy.get( + '#test-scatter-dismiss-preview svg .observed-window-range[data-source="overlay"]', + ).should('exist'); cy.get('[data-testid="dismiss-preview"]').click(); cy.get('#test-scatter-dismiss-preview svg .unofficial-overlay-pt').should('not.exist'); cy.get('#test-scatter-dismiss-preview svg .overlay-roofline-path').should('not.exist'); + cy.get( + '#test-scatter-dismiss-preview svg .observed-window-range[data-source="overlay"]', + ).should('not.exist'); }); }); diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 1ce282c2..ecd5f04a 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -44,6 +44,12 @@ const percentileLadder = (prefix: string, base: number): Record const agenticMetrics = (conc: number): Record => { const scale = conc / 16; const itl = 0.011 * scale; + const p75Ttft = 0.4 * scale * 1.2; + const p90Ttft = 0.4 * scale * 1.5; + const p75E2e = 8 * scale * 1.2; + const p90E2e = 8 * scale * 1.5; + const p75Interactivity = 1 / (itl * 1.2); + const p90Interactivity = 1 / (itl * 1.5); return { ...percentileLadder('ttft', 0.4 * scale), ...percentileLadder('tpot', 0.012 * scale), @@ -58,6 +64,25 @@ const agenticMetrics = (conc: number): Record => { output_tput_per_gpu: 210, input_tput_per_gpu: 740, total_tput_tps: 7600 * conc * 0.05, + observed_window_seconds: 600, + observed_window_expected_count: 6, + observed_window_count: 6, + observed_window_min_requests: Math.max(12, Math.round(conc * 1.25)), + root_trajectory_count: Math.max(3, Math.round(conc / 4)), + root_trajectory_kish_effective_count: Math.max(1.8, conc / 8), + root_trajectory_largest_share: Math.min(0.56, 8 / conc), + observed_window_p75_ttft_min: p75Ttft * 0.55, + observed_window_p75_ttft_max: p75Ttft * 1.45, + observed_window_p90_ttft_min: p90Ttft * 0.5, + observed_window_p90_ttft_max: p90Ttft * 1.5, + observed_window_p75_e2el_min: p75E2e * 0.75, + observed_window_p75_e2el_max: p75E2e * 1.25, + observed_window_p90_e2el_min: p90E2e * 0.7, + observed_window_p90_e2el_max: p90E2e * 1.3, + observed_window_p75_intvty_min: p75Interactivity * 0.72, + observed_window_p75_intvty_max: p75Interactivity * 1.28, + observed_window_p90_intvty_min: p90Interactivity * 0.68, + observed_window_p90_intvty_max: p90Interactivity * 1.32, }; }; @@ -131,12 +156,21 @@ const fixedSequenceBenchmarks = agenticBenchmarks.map((row, index) => ({ benchmark_type: 'single_turn', })); +const interceptDashboardMetadata = () => { + cy.intercept('GET', '/api/v1/workflow-info*', { + body: { runs: [], changelogs: [], configs: [], runConfigs: [] }, + }).as('workflowInfo'); + cy.intercept('GET', '/api/v1/trace-availability*', { body: {} }).as('traceAvailability'); +}; + const interceptAgenticData = () => { + interceptDashboardMetadata(); cy.intercept('GET', '/api/v1/availability', { body: agenticAvailability }).as('availability'); cy.intercept('GET', '/api/v1/benchmarks*', { body: agenticBenchmarks }).as('benchmarks'); }; const interceptFixedSequenceData = () => { + interceptDashboardMetadata(); cy.intercept('GET', '/api/v1/availability', { body: agenticAvailability }).as('availability'); cy.intercept('GET', '/api/v1/benchmarks*', { body: fixedSequenceBenchmarks }).as('benchmarks'); }; @@ -191,6 +225,56 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); }); + it('renders observed-window ranges with an explicit non-inferential caveat', () => { + cy.get( + '[data-testid="inference-chart-display"] [data-testid="observed-window-range"][data-source="official"]', + ).should('have.length.greaterThan', 0); + cy.get('[data-testid="observed-window-range-key"]') + .should('contain.text', 'Observed 10-minute range') + .and('contain.text', 'not a confidence interval'); + + cy.get('[data-testid="scatter-graph"] svg .dot-group .visible-shape') + .first() + .click({ force: true }); + cy.get('[data-chart-tooltip]:visible') + .should('contain.text', 'Observed 10-minute range') + .and('contain.text', 'not a confidence interval or rerun prediction') + .and('contain.text', 'Kish-effective root coverage'); + }); + + it('keeps observed-window ranges anchored through chart zoom', () => { + cy.get( + '[data-testid="inference-chart-display"] [data-testid="observed-window-range"][data-source="official"] .observed-window-range-stem', + ) + .first() + .then(($stem) => { + const before = [$stem.attr('x1'), $stem.attr('x2')]; + cy.get('[data-testid="scatter-graph"] svg') + .first() + .then(($svg) => { + const svg = $svg[0]; + const bounds = svg.getBoundingClientRect(); + svg.dispatchEvent( + new WheelEvent('wheel', { + deltaY: -240, + clientX: bounds.x + bounds.width / 2, + clientY: bounds.y + bounds.height / 2, + shiftKey: true, + bubbles: true, + cancelable: true, + }), + ); + }); + cy.wait(300); + cy.wrap($stem).should(($zoomedStem) => { + expect( + [$zoomedStem.attr('x1'), $zoomedStem.attr('x2')], + 'range geometry after zoom', + ).not.to.deep.equal(before); + }); + }); + }); + it('switches the x-axis to E2E Latency and updates the heading', () => { cy.get('[data-testid="x-axis-mode-e2e"]').click(); cy.get('[data-testid="x-axis-mode-e2e"]').should('have.attr', 'aria-selected', 'true'); @@ -371,6 +455,11 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () }); expect(total).to.be.greaterThan(0); }); + cy.get( + '[data-testid="inference-chart-display"] [data-testid="observed-window-range"][data-source="overlay"]', + ) + .should('have.length.greaterThan', 0) + .and('have.attr', 'data-run-index', '0'); }); it('normalized-e2e mode shows suppression banner for unofficial-run overlays', () => { diff --git a/packages/app/src/components/inference/hooks/useChartData.test.ts b/packages/app/src/components/inference/hooks/useChartData.test.ts index c4998add..497c3a07 100644 --- a/packages/app/src/components/inference/hooks/useChartData.test.ts +++ b/packages/app/src/components/inference/hooks/useChartData.test.ts @@ -5,8 +5,11 @@ import { dedupeRowsToLatestPerConfig, filterByGPU, flipRooflineDirection, + remapChartPoint, } from './useChartData'; +import type { InferenceData } from '@/components/inference/types'; + interface DedupeInput { id: number; hardware: string; @@ -151,3 +154,25 @@ describe('flipRooflineDirection', () => { } }); }); + +describe('remapChartPoint', () => { + it('preserves the selected metric roof flag while flattening its y value', () => { + const point = { + x: 1, + y: 2, + hwKey: 'h100', + date: '2026-06-01', + tp: 100, + conc: 1, + precision: 'fp8', + p90_ttft: 3, + tpPerGpu: { y: 42, roof: true }, + } as InferenceData; + + expect(remapChartPoint(point, 'tpPerGpu', 'p90_ttft')).toMatchObject({ + x: 3, + y: 42, + roof: true, + }); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index 197fcd6b..7b4b897b 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -29,6 +29,7 @@ import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { calculateCostsForGpus, calculatePowerForGpus } from '@/lib/utils'; import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; +import { withObservedWindowRange } from '@/components/inference/utils/observed-window-range'; import { applyQuickFilters, computeAvailableQuickFilters, @@ -139,6 +140,32 @@ export function flipRooflineDirection(dir: RooflineDirection): RooflineDirection return FLIP_MAP[dir]; } +/** + * Remap a transformed benchmark point onto the currently selected chart axes. + * Keep the selected y-metric's roofline marker on the flattened point: chart + * consumers read this marker after the metric object has been reduced to `y`. + */ +export function remapChartPoint( + point: InferenceData, + metricKey: YAxisMetricKey, + xAxisField: keyof AggDataEntry, + isOnE2eFrontier?: boolean, +): InferenceData { + const metric = point[metricKey] as { y: number; roof: boolean } | undefined; + const xCandidate = (point as Partial)[xAxisField]; + + return withObservedWindowRange( + { + ...point, + x: typeof xCandidate === 'number' ? xCandidate : point.x, + y: metric?.y ?? point.y, + roof: metric?.roof ?? false, + isOnE2eFrontier, + }, + String(xAxisField), + ); +} + /** The dedup key fields a chart series is identified by. */ interface DedupeRow { hardware: string; @@ -526,25 +553,11 @@ export function useChartData( ? filteredData .filter((d) => metricKey in d) .map((d: InferenceData) => { - const yValue = (d[metricKey] as { y: number })?.y ?? d.y; - const roof = (d[metricKey] as { roof: boolean })?.roof ?? false; - // xAxisField is `keyof AggDataEntry`; InferenceData embeds those - // fields via `Partial>`, so a typed - // accessor catches a future field rename (silent fallthrough to - // d.x would otherwise mask the regression). - const xCandidate = (d as Partial)[xAxisField]; - const xValue = typeof xCandidate === 'number' ? xCandidate : d.x; const isOnE2eFrontier = e2eParetoSet === null ? undefined : isPersistedBenchmarkId(d.id) && e2eParetoSet.has(d.id); - return { - ...d, - x: xValue, - y: yValue, - roof, - isOnE2eFrontier, - }; + return remapChartPoint(d, metricKey, xAxisField, isOnE2eFrontier); }) // When TTFT is on the x-axis, apply the latency limit to filter // overload outliers (fixed-seq conc=2048 rows with TTFT > 60s that diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8b45e0f7..f7cd530a 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -219,6 +219,32 @@ export interface AggDataEntry { total_prompt_tokens?: number; /** Total generated (output) tokens. */ total_generation_tokens?: number; + /** Width of each non-overlapping descriptive window, in seconds. */ + observed_window_seconds?: number; + /** Configured number of complete windows in the measurement duration. */ + observed_window_expected_count?: number; + /** Number of those windows containing at least one successful request. */ + observed_window_count?: number; + /** Smallest successful-request count among observed windows. */ + observed_window_min_requests?: number; + /** Distinct root source trajectories represented by all requests. */ + root_trajectory_count?: number; + /** Kish diversity count from root-trajectory request shares (not a statistical ESS). */ + root_trajectory_kish_effective_count?: number; + /** Fraction of requests belonging to the most represented root trajectory. */ + root_trajectory_largest_share?: number; + observed_window_p75_ttft_min?: number; + observed_window_p75_ttft_max?: number; + observed_window_p90_ttft_min?: number; + observed_window_p90_ttft_max?: number; + observed_window_p75_e2el_min?: number; + observed_window_p75_e2el_max?: number; + observed_window_p90_e2el_min?: number; + observed_window_p90_e2el_max?: number; + observed_window_p75_intvty_min?: number; + observed_window_p75_intvty_max?: number; + observed_window_p90_intvty_min?: number; + observed_window_p90_intvty_max?: number; } /** @@ -243,7 +269,12 @@ export interface InferenceData extends Partial ({ hwKey, precision, x, y, tp, conc: 16, framework: 'vllm' }) as unknown as InferenceData; +const rangedPoint = ( + hwKey: string, + x: number, + y: number, + min: number, + max: number, +): InferenceData => ({ + ...point(hwKey, 'fp8', x, y, 8), + observedXMin: min, + observedXMax: max, + observed_window_seconds: 600, + observed_window_count: 6, +}); + // h100 owns both axis extremes so hiding b200 / showing fp4 keeps the niced // domains identical — exactly the toggle case that must not rebuild. const POINTS: InferenceData[] = [ @@ -139,6 +154,7 @@ function baseOverlayState() { // ── Harness ────────────────────────────────────────────────────────────────── function mountChart(props?: Partial[0]>) { let forceUpdate: () => void = noop; + let currentProps = props ?? {}; function Harness() { // ScatterGraph and D3Chart are React.memo'd; mocked context hooks bypass // React's context subscription, so re-renders are driven through a @@ -154,7 +170,7 @@ function mountChart(props?: Partial[0]>) { chartDefinition: CHART_DEFINITION, transitionDuration: 0, caption: `v${version}`, - ...props, + ...currentProps, }); } @@ -167,6 +183,10 @@ function mountChart(props?: Partial[0]>) { return { container, rerender: () => act(() => forceUpdate()), + updateProps: (next: Partial[0]>) => { + currentProps = { ...currentProps, ...next }; + act(() => forceUpdate()); + }, unmount: () => { act(() => root.unmount()); container.remove(); @@ -215,6 +235,51 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('renders observed ranges only for Pareto-front points and fits their x extent', () => { + const data = [ + rangedPoint('h100', 1, 50, 0.5, 1.8), + rangedPoint('h100', 2, 100, 1.2, 8), + // Below the running upper-right envelope, so it must not get a whisker. + rangedPoint('h100', 1.5, 40, 0.8, 6), + ]; + const { container, unmount } = mountChart({ + data, + chartDefinition: { + ...CHART_DEFINITION, + y_roofline: 'upper_right', + } as unknown as ChartDefinition, + }); + + const ranges = [ + ...container.querySelectorAll('.observed-window-range[data-source="official"]'), + ]; + expect(ranges).toHaveLength(2); + expect(ranges.every((range) => range.style.pointerEvents === 'none')).toBe(true); + expect( + ranges.some( + (range) => + (range as SVGGElement & { __data__: { point: InferenceData } }).__data__.point.x === 1.5, + ), + ).toBe(false); + + const wideRange = ranges.find( + (range) => + (range as SVGGElement & { __data__: { point: InferenceData } }).__data__.point.x === 2, + )!; + const stem = wideRange.querySelector('.observed-window-range-stem')!; + const pointDot = dotGroups(container).find( + (dot) => (dot as SVGGElement & { __data__: InferenceData }).__data__.x === 2, + )!; + const pointX = Number( + pointDot.getAttribute('transform')?.match(/translate\((?[^,]+)/u)?.groups?.x, + ); + expect(Number(stem.getAttribute('x2'))).toBeGreaterThan(pointX); + // The observed max participates in the domain, so its cap remains inside + // the 800px chart rather than being clipped at the right edge. + expect(Number(stem.getAttribute('x2'))).toBeLessThan(800); + unmount(); + }); + it('hides a toggled-off hw via opacity without rebuilding the chart', () => { const { container, rerender, unmount } = mountChart(); const buildsAfterMount = rebuildCount(); @@ -377,6 +442,49 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('renders and removes overlay observed ranges through the overlay path', () => { + const overlayPoints = [ + rangedPoint('h100', 30, 250, 20, 36), + rangedPoint('h100', 35, 300, 28, 48), + ].map((p) => ({ + ...p, + run_url: 'https://github.com/o/r/actions/runs/123', + })); + overlayState.current = { + ...baseOverlayState(), + isUnofficialRun: true, + activeOverlayHwTypes: new Set(['h100']), + allOverlayHwTypes: new Set(['h100']), + runIndexByUrl: { 'https://github.com/o/r/actions/runs/123': 0 }, + unofficialRunInfos: [ + { id: '123', branch: 'test-branch', url: 'https://github.com/o/r/actions/runs/123' }, + ], + }; + const overlayData = { + data: overlayPoints, + hardwareConfig: HARDWARE_CONFIG, + } as unknown as Parameters[0]['overlayData']; + const { container, updateProps, unmount } = mountChart({ + overlayData, + chartDefinition: { + ...CHART_DEFINITION, + y_roofline: 'upper_right', + } as unknown as ChartDefinition, + }); + + const ranges = [ + ...container.querySelectorAll('.observed-window-range[data-source="overlay"]'), + ]; + expect(ranges).toHaveLength(2); + expect(ranges.every((range) => range.getAttribute('stroke') === overlayRunColor(0))).toBe(true); + + updateProps({ overlayData: undefined }); + expect( + container.querySelectorAll('.observed-window-range[data-source="overlay"]'), + ).toHaveLength(0); + unmount(); + }); + it('applies quick filters to unofficial-run overlay markers', () => { const overlayPoints = [point('h100', 'fp8', 30, 300, 2), point('h100', 'fp8', 35, 350, 4)].map( (p) => ({ ...p, run_url: 'https://github.com/o/r/actions/runs/123' }), diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 16e3ab6d..1124f81c 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -261,6 +261,46 @@ interface ScaleConfigValue { nice: boolean; _isLog?: boolean; } + +interface ObservedRangeMark { + key: string; + point: InferenceData & { observedXMin: number; observedXMax: number }; + hw: string; + precision: string; + source: 'official' | 'overlay'; + color: string; + visible: boolean; + runIndex?: number; +} + +const hasObservedWindowRange = ( + point: InferenceData, +): point is InferenceData & { observedXMin: number; observedXMax: number } => + (point.observed_window_count ?? 0) >= 2 && + typeof point.observedXMin === 'number' && + Number.isFinite(point.observedXMin) && + point.observedXMin > 0 && + typeof point.observedXMax === 'number' && + Number.isFinite(point.observedXMax) && + point.observedXMax > point.observedXMin; + +const observedRangeLegendLabel = ( + locale: 'en' | 'zh', + windowSeconds: number | undefined, +): string => { + if (!windowSeconds) { + return locale === 'zh' + ? '观测时间窗口范围——并非置信区间' + : 'Observed window range — not a confidence interval'; + } + const minutes = windowSeconds / 60; + if (locale === 'zh') { + const duration = Number.isInteger(minutes) ? `${minutes} 分钟` : `${windowSeconds} 秒`; + return `${duration}观测范围——并非置信区间`; + } + const duration = Number.isInteger(minutes) ? `${minutes}-minute` : `${windowSeconds}-second`; + return `Observed ${duration} range — not a confidence interval`; +}; const isSameScaleConfig = (a: ScaleConfigValue, b: ScaleConfigValue): boolean => a.type === b.type && a.nice === b.nice && @@ -935,6 +975,51 @@ const ScatterGraph = React.memo( [allPointLabelsByKey], ); + // Window ranges are a descriptive annotation on the Pareto frontier, not + // additional observations and not an alternate frontier. Keeping this + // list sourced from `rooflines` / `overlayRooflines` guarantees a + // dominated scatter point can never stretch the axis merely because it + // carries diagnostics. + const visibleFrontierPoints = useMemo(() => { + const points: InferenceData[] = []; + for (const [key, front] of Object.entries(rooflines)) { + const hw = key.split('_').slice(0, -1).join('_'); + const precision = key.split('_').pop()!; + if (effectiveActiveHwTypes.has(hw) && selectedPrecisions.includes(precision)) { + points.push(...front); + } + } + for (const group of Object.values(overlayRooflines)) { + if (activeOverlayHwTypes.has(group.hwKey)) points.push(...group.points); + } + return points; + }, [ + rooflines, + overlayRooflines, + effectiveActiveHwTypes, + selectedPrecisions, + activeOverlayHwTypes, + ]); + + const observedRangePoints = useMemo( + () => visibleFrontierPoints.filter(hasObservedWindowRange), + [visibleFrontierPoints], + ); + + // A mixed-duration comparison gets a generic key rather than claiming one + // window width applies to every point. Current AgentX artifacts use 600s. + const observedRangeWindowSeconds = useMemo(() => { + const durations = new Set( + observedRangePoints + .map((point) => point.observed_window_seconds) + .filter( + (seconds): seconds is number => + typeof seconds === 'number' && Number.isFinite(seconds) && seconds > 0, + ), + ); + return durations.size === 1 ? [...durations][0] : undefined; + }, [observedRangePoints]); + // Ref for trackedConfigIds (needs to be current at event time inside D3 handlers) const trackedConfigIdsRef = useRef(trackedConfigIds); trackedConfigIdsRef.current = trackedConfigIds; @@ -966,7 +1051,10 @@ const ScatterGraph = React.memo( const ext = xExtentOverride ?? (visiblePoints.length > 0 - ? (d3.extent(visiblePoints, (d) => d.x) as [number, number]) + ? (d3.extent([ + ...visiblePoints.map((point) => point.x), + ...observedRangePoints.flatMap((point) => [point.observedXMin, point.observedXMax]), + ]) as [number, number]) : ([0, 100] as [number, number])); let useLog = false; @@ -986,7 +1074,15 @@ const ScatterGraph = React.memo( nice: niceAxes, _isLog: useLog, }; - }, [visiblePoints, isInputTputMetric, xLabel, scaleType, niceAxes, xExtentOverride]); + }, [ + visiblePoints, + observedRangePoints, + isInputTputMetric, + xLabel, + scaleType, + niceAxes, + xExtentOverride, + ]); const xScaleConfig = useStableValue(xScaleConfigRaw, isSameScaleConfig); const yScaleConfigRaw = useMemo(() => { @@ -1059,6 +1155,20 @@ const ScatterGraph = React.memo( [effectiveActiveHwTypes, selectedPrecisions], ); + const observedRangeOpacity = useCallback( + (el: SVGGElement): number => { + const hw = el.dataset.hwKey; + const precision = el.dataset.precision; + if (!hw || !precision) return 0; + const active = + el.dataset.source === 'overlay' + ? activeOverlayHwTypes.has(hw) + : effectiveActiveHwTypes.has(hw); + return active && selectedPrecisions.includes(precision) ? 0.42 : 0; + }, + [activeOverlayHwTypes, effectiveActiveHwTypes, selectedPrecisions], + ); + // --- Interaction state ref --- // Latest visibility predicates, color resolvers, and active sets — read by // long-lived D3 closures (layer renders, zoom handlers, hover handlers). @@ -1115,13 +1225,20 @@ const ScatterGraph = React.memo( if (!isRooflineVisible(this)) return 0; return this.dataset.hwKey === hwKey ? null : '0.15'; }); + root + .selectAll('.observed-window-range[data-source="official"]') + .style('opacity', function () { + const base = observedRangeOpacity(this); + if (base === 0) return 0; + return this.dataset.hwKey === hwKey ? base : 0.06; + }); root .selectAll('.parallelism-label, .line-label') .style('opacity', function () { return labelOpacityForHover((this as SVGGElement).dataset, hwKey); }); }, - [isPointVisible, isRooflineVisible], + [isPointVisible, isRooflineVisible, observedRangeOpacity], ); const handleLegendHoverEnd = useCallback(() => { @@ -1134,6 +1251,9 @@ const ScatterGraph = React.memo( root.selectAll('.roofline-path').style('opacity', function () { return isRooflineVisible(this) ? 1 : 0; }); + root.selectAll('.observed-window-range').style('opacity', function () { + return observedRangeOpacity(this); + }); root .selectAll('.parallelism-label, .line-label') .style('opacity', function () { @@ -1143,7 +1263,13 @@ const ScatterGraph = React.memo( selectedPrecisions, ); }); - }, [isPointVisible, isRooflineVisible, effectiveActiveHwTypes, selectedPrecisions]); + }, [ + isPointVisible, + isRooflineVisible, + observedRangeOpacity, + effectiveActiveHwTypes, + selectedPrecisions, + ]); // --- Zoom config --- const eventPrefix = chartDefinition.chartType === 'e2e' ? 'latency' : 'interactivity'; @@ -1403,6 +1529,105 @@ const ScatterGraph = React.memo( .style('transition', 'opacity 150ms ease') .style('opacity', (d) => (d.visible ? 1 : 0)); + // Thin, capped horizontal whiskers summarize the range observed + // across non-overlapping windows in this one run. They deliberately + // live in the roofline layer (behind points), carry no pointer + // events, and are never used to compute the Pareto frontier. + const observedRangeMarks: ObservedRangeMark[] = []; + for (const [key, front] of Object.entries(rooflines)) { + const hw = key.split('_').slice(0, -1).join('_'); + const precision = key.split('_').pop()!; + const visible = + ir.effectiveActiveHwTypes.has(hw) && ir.selectedPrecisions.includes(precision); + front.forEach((point, index) => { + if (!hasObservedWindowRange(point)) return; + observedRangeMarks.push({ + key: `official-${key}-${point.date}-${point.id ?? 'na'}-${point.conc}-${point.x}-${point.y}-${index}`, + point, + hw, + precision, + source: 'official', + color: ir.getCssColor(ir.resolveColor(hw)), + visible, + }); + }); + } + if (overlayData) { + for (const [key, group] of Object.entries(overlayRooflines)) { + if (!overlayData.hardwareConfig[group.hwKey]) continue; + group.points.forEach((point, index) => { + if (!hasObservedWindowRange(point)) return; + observedRangeMarks.push({ + key: `overlay-${key}-${point.date}-${point.id ?? 'na'}-${point.conc}-${point.x}-${point.y}-${index}`, + point, + hw: group.hwKey, + precision: point.precision, + source: 'overlay', + color: overlayRunColor(group.runIndex), + visible: + ir.activeOverlayHwTypes.has(group.hwKey) && + ir.selectedPrecisions.includes(point.precision), + runIndex: group.runIndex, + }); + }); + } + } + + const observedRanges = rooflinesLayer + .selectAll('.observed-window-range') + .data(observedRangeMarks, (mark) => mark.key) + .join( + (enter) => { + const group = enter.append('g').attr('class', 'observed-window-range'); + group.append('line').attr('class', 'observed-window-range-stem'); + group.append('line').attr('class', 'observed-window-range-cap-min'); + group.append('line').attr('class', 'observed-window-range-cap-max'); + return group; + }, + (update) => update, + (exit) => exit.remove(), + ) + .attr('data-testid', 'observed-window-range') + .attr('data-source', (mark) => mark.source) + .attr('data-hw-key', (mark) => mark.hw) + .attr('data-precision', (mark) => mark.precision) + .attr('data-run-index', (mark) => mark.runIndex ?? null) + .attr('aria-hidden', 'true') + .attr('fill', 'none') + .attr('stroke', (mark) => mark.color) + .attr('stroke-width', 1) + .attr('stroke-linecap', 'round') + .style('pointer-events', 'none') + .style('transition', 'opacity 150ms ease') + .style('opacity', (mark) => (mark.visible ? 0.42 : 0)); + + observedRanges.selectAll('line').attr('vector-effect', 'non-scaling-stroke'); + observedRanges.each(function (mark) { + const group = d3.select(this); + const minX = xScale(mark.point.observedXMin); + const maxX = xScale(mark.point.observedXMax); + const y = yScale(mark.point.y); + group + .select('.observed-window-range-stem') + .attr('x1', minX) + .attr('x2', maxX) + .attr('y1', y) + .attr('y2', y); + group + .select('.observed-window-range-cap-min') + .attr('x1', minX) + .attr('x2', minX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + group + .select('.observed-window-range-cap-max') + .attr('x1', maxX) + .attr('x2', maxX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + }); + observedRanges.lower(); + // Parallelism labels interface LabelSeg { segKey: string; @@ -1854,6 +2079,35 @@ const ScatterGraph = React.memo( } }); + // Keep observed-window whiskers anchored to the full-run point while + // zooming. End caps stay a fixed seven CSS pixels tall. + zoomGroup + .selectAll('.observed-window-range') + .each(function (mark) { + const group = d3.select(this); + const minX = newXScale(mark.point.observedXMin); + const maxX = newXScale(mark.point.observedXMax); + const y = newYScale(mark.point.y); + group + .select('.observed-window-range-stem') + .attr('x1', minX) + .attr('x2', maxX) + .attr('y1', y) + .attr('y2', y); + group + .select('.observed-window-range-cap-min') + .attr('x1', minX) + .attr('x2', minX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + group + .select('.observed-window-range-cap-max') + .attr('x1', maxX) + .attr('x2', maxX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + }); + // Update gradient coordinates if (showGradientLabels) { Object.entries(allPointLabelsByKey).forEach(([key, pointLabels]) => { @@ -2683,6 +2937,16 @@ const ScatterGraph = React.memo( } }); + // Observed-window ranges follow the same visibility/recolor path as + // their frontier series without touching their zoomed geometry. + zoomGroup.selectAll('.observed-window-range').each(function () { + const el = d3.select(this); + el.style('opacity', observedRangeOpacity(this)); + if (this.dataset.source === 'official' && this.dataset.hwKey) { + el.attr('stroke', ir.getCssColor(ir.resolveColor(this.dataset.hwKey))); + } + }); + // Parallelism / line labels: visibility via data attributes (mirrors // handleLegendHoverEnd). Placement-level updates happen below. zoomGroup @@ -2740,6 +3004,7 @@ const ScatterGraph = React.memo( showGradientLabels, showLineLabels, gradientColorByPoint, + observedRangeOpacity, ]); // D3 custom layers are keyed additions, so removing the overlay layer from @@ -2749,7 +3014,11 @@ const ScatterGraph = React.memo( if (overlayData) return; const svg = chartRef.current?.getSvgElement?.(); if (!svg) return; - d3.select(svg).selectAll('.unofficial-overlay-pt, .overlay-roofline-path').remove(); + d3.select(svg) + .selectAll( + '.unofficial-overlay-pt, .overlay-roofline-path, .observed-window-range[data-source="overlay"]', + ) + .remove(); }, [overlayData]); // Dismiss tooltip on filter changes @@ -2937,6 +3206,32 @@ const ScatterGraph = React.memo( })), ]} disableActiveSort={false} + keyIndicators={ + observedRangePoints.length > 0 ? ( +
+ + {observedRangeLegendLabel(locale, observedRangeWindowSeconds)} +
+ ) : undefined + } isLegendExpanded={isLegendExpanded} onExpandedChange={(expanded) => { setIsLegendExpanded(expanded); diff --git a/packages/app/src/components/inference/utils.test.ts b/packages/app/src/components/inference/utils.test.ts index 35e48ad2..7ff03720 100644 --- a/packages/app/src/components/inference/utils.test.ts +++ b/packages/app/src/components/inference/utils.test.ts @@ -233,6 +233,24 @@ describe('processOverlayChartData', () => { expect(result[0].x).toBe(0.12); }); + it('stamps the observed bounds matching the overlay x-axis metric', () => { + const data = [ + pt({ + tpPerGpu: { y: 42, roof: false }, + p90_ttft: 2, + observed_window_count: 6, + observed_window_p90_ttft_min: 1.4, + observed_window_p90_ttft_max: 4.1, + observed_window_p75_e2el_min: 40, + observed_window_p75_e2el_max: 80, + } as any), + ]; + const result = processOverlayChartData(data, 'e2e', 'y_tpPerGpu', 'p90_ttft'); + + expect(result[0].observedXMin).toBe(1.4); + expect(result[0].observedXMax).toBe(4.1); + }); + it('filters e2e TTFT outliers exceeding y_latency_limit', () => { const data = [ pt({ tpPerGpu: { y: 10, roof: false }, p90_ttft: 0.5, median_e2el: 1 } as any), diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index 2b8074d9..c54d26b6 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -7,6 +7,7 @@ import chartDefinitions from '@/components/inference/inference-chart-config.json'; import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; +import { withObservedWindowRange } from '@/components/inference/utils/observed-window-range'; import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; @@ -127,7 +128,7 @@ export function processOverlayChartData( .map((d: InferenceData) => { const yValue = (d[metricKey] as { y: number })?.y ?? d.y; const xValue = (d as any)[xAxisField] ?? d.x; - return { ...d, x: xValue, y: yValue }; + return withObservedWindowRange({ ...d, x: xValue, y: yValue }, xAxisField); }) .filter( (d) => !isTtftX || isAgentic || !chartDef.y_latency_limit || d.x <= chartDef.y_latency_limit, diff --git a/packages/app/src/components/inference/utils/observed-window-range.test.ts b/packages/app/src/components/inference/utils/observed-window-range.test.ts new file mode 100644 index 00000000..50205535 --- /dev/null +++ b/packages/app/src/components/inference/utils/observed-window-range.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest'; + +import type { InferenceData } from '@/components/inference/types'; + +import { observedWindowRangeForXAxis, withObservedWindowRange } from './observed-window-range'; + +describe('observedWindowRangeForXAxis', () => { + it('selects bounds matching the resolved x-axis field', () => { + const point = { + observed_window_count: 6, + observed_window_p90_ttft_min: 1.4, + observed_window_p90_ttft_max: 4.1, + observed_window_p75_e2el_min: 40, + observed_window_p75_e2el_max: 80, + }; + + expect(observedWindowRangeForXAxis(point, 'p90_ttft')).toEqual({ min: 1.4, max: 4.1 }); + expect(observedWindowRangeForXAxis(point, 'p75_e2el')).toEqual({ min: 40, max: 80 }); + }); + + it('requires at least two observed windows', () => { + expect( + observedWindowRangeForXAxis( + { + observed_window_count: 1, + observed_window_p90_ttft_min: 1, + observed_window_p90_ttft_max: 2, + }, + 'p90_ttft', + ), + ).toBeNull(); + }); + + it('rejects missing, non-positive, reversed, and degenerate bounds', () => { + expect(observedWindowRangeForXAxis({ observed_window_count: 6 }, 'p90_ttft')).toBeNull(); + expect( + observedWindowRangeForXAxis( + { + observed_window_count: 6, + observed_window_p90_ttft_min: 0, + observed_window_p90_ttft_max: 2, + }, + 'p90_ttft', + ), + ).toBeNull(); + expect( + observedWindowRangeForXAxis( + { + observed_window_count: 6, + observed_window_p90_ttft_min: 2, + observed_window_p90_ttft_max: 2, + }, + 'p90_ttft', + ), + ).toBeNull(); + }); +}); + +describe('withObservedWindowRange', () => { + it('stamps chart-ready bounds and clears stale bounds when the x field changes', () => { + const point = { + x: 2, + y: 100, + date: '2026-07-21', + tp: 8, + conc: 16, + precision: 'fp4', + hwKey: 'b300-vllm', + tpPerGpu: { y: 100, roof: true }, + tpPerMw: { y: 1, roof: true }, + costh: { y: 1, roof: true }, + costn: { y: 1, roof: true }, + costr: { y: 1, roof: true }, + costhi: { y: 1, roof: true }, + costni: { y: 1, roof: true }, + costri: { y: 1, roof: true }, + observed_window_count: 6, + observed_window_p90_ttft_min: 1, + observed_window_p90_ttft_max: 3, + } as InferenceData; + + const stamped = withObservedWindowRange(point, 'p90_ttft'); + expect(stamped.observedXMin).toBe(1); + expect(stamped.observedXMax).toBe(3); + + const cleared = withObservedWindowRange(stamped, 'median_ttft'); + expect(cleared.observedXMin).toBeUndefined(); + expect(cleared.observedXMax).toBeUndefined(); + }); +}); diff --git a/packages/app/src/components/inference/utils/observed-window-range.ts b/packages/app/src/components/inference/utils/observed-window-range.ts new file mode 100644 index 00000000..93d078a0 --- /dev/null +++ b/packages/app/src/components/inference/utils/observed-window-range.ts @@ -0,0 +1,41 @@ +import type { AggDataEntry, InferenceData } from '@/components/inference/types'; + +export interface ObservedWindowRange { + min: number; + max: number; +} + +/** + * Resolve the descriptive window bounds matching the chart's current x field. + * Missing, degenerate, and single-window diagnostics intentionally render no + * whisker: there is no within-run range to communicate in those cases. + */ +export function observedWindowRangeForXAxis( + point: Partial, + xAxisField: string, +): ObservedWindowRange | null { + if ((point.observed_window_count ?? 0) < 2) return null; + const values = point as unknown as Record; + const min = values[`observed_window_${xAxisField}_min`]; + const max = values[`observed_window_${xAxisField}_max`]; + if ( + typeof min !== 'number' || + typeof max !== 'number' || + !Number.isFinite(min) || + !Number.isFinite(max) || + min <= 0 || + max <= min + ) { + return null; + } + return { min, max }; +} + +export function withObservedWindowRange(point: InferenceData, xAxisField: string): InferenceData { + const range = observedWindowRangeForXAxis(point, xAxisField); + return { + ...point, + observedXMin: range?.min, + observedXMax: range?.max, + }; +} diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index 893819f5..301bb488 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -71,6 +71,21 @@ function tooltipConfig(overrides: Partial = {}): TooltipConfig { }; } +function observedRangePoint(overrides: Partial = {}): InferenceData { + return pt({ + benchmark_type: 'agentic_traces', + observedXMin: 1.416, + observedXMax: 4.114, + observed_window_seconds: 600, + observed_window_expected_count: 6, + observed_window_count: 6, + observed_window_min_requests: 21, + root_trajectory_count: 3, + root_trajectory_kish_effective_count: 1.69, + ...overrides, + }); +} + // =========================================================================== // getPointLabel // =========================================================================== @@ -425,6 +440,39 @@ describe('generateTooltipContent', () => { expect(html).toContain('Track Over Time'); expect(html).not.toContain('Untrack Over Time'); }); + + it('labels the observed window range as descriptive, not inferential', () => { + const html = generateTooltipContent(tooltipConfig({ data: observedRangePoint() })); + + expect(html).toContain('Observed 10-minute range: 1.416–4.114'); + expect(html).toContain( + '6 non-overlapping windows; not a confidence interval or rerun prediction.', + ); + expect(html).toContain('Root trajectories: 3'); + expect(html).toContain('Kish-effective root coverage: 1.69'); + expect(html).toContain('not a statistical effective sample size'); + expect(html).toContain('Smallest window: 21 successful requests'); + }); + + it('renders natural Chinese observed-range caveats', () => { + const html = generateTooltipContent( + tooltipConfig({ locale: 'zh', data: observedRangePoint() }), + ); + + expect(html).toContain('10 分钟观测范围: 1.416–4.114'); + expect(html).toContain('基于 6 个互不重叠的时间窗口；并非置信区间，也不预测复跑结果。'); + expect(html).toContain('根轨迹数: 3'); + expect(html).toContain('Kish 有效根轨迹覆盖数: 1.69'); + expect(html).toContain('并非统计有效样本量'); + expect(html).toContain('最小时间窗口: 21个成功请求'); + }); + + it('omits an observed range when fewer than two windows are available', () => { + const html = generateTooltipContent( + tooltipConfig({ data: observedRangePoint({ observed_window_count: 1 }) }), + ); + expect(html).not.toContain('observed-window-range-tooltip'); + }); }); // =========================================================================== @@ -506,6 +554,14 @@ describe('generateOverlayTooltipContent', () => { expect(html).not.toContain('Offload Type'); expect(html).not.toContain('CPU Cache Hit Rate'); }); + + it('shows the same observed-range caveat for unofficial overlays', () => { + const html = generateOverlayTooltipContent(overlayConfig({ data: observedRangePoint() })); + + expect(html).toContain('Observed 10-minute range: 1.416–4.114'); + expect(html).toContain('not a confidence interval or rerun prediction'); + expect(html).toContain('Kish-effective root coverage'); + }); }); // =========================================================================== diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 9226381d..7a5d97fe 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -99,6 +99,106 @@ export const fmt = (v: number): string => { return String(rounded); }; +const OBSERVED_RANGE_STRINGS = { + en: { + genericTitle: 'Observed window range', + title: (duration: string) => `Observed ${duration} range`, + completeWindows: (count: number) => + `${count} non-overlapping windows; not a confidence interval or rerun prediction.`, + partialWindows: (count: number, expected: number) => + `${count} of ${expected} non-overlapping windows contained successful requests; not a confidence interval or rerun prediction.`, + rootTrajectories: 'Root trajectories', + kishCoverage: 'Kish-effective root coverage', + kishExplanation: 'coverage diversity, not a statistical effective sample size', + smallestWindow: 'Smallest window', + successfulRequests: 'successful requests', + }, + zh: { + genericTitle: '观测时间窗口范围', + title: (duration: string) => `${duration}观测范围`, + completeWindows: (count: number) => + `基于 ${count} 个互不重叠的时间窗口；并非置信区间，也不预测复跑结果。`, + partialWindows: (count: number, expected: number) => + `${expected} 个互不重叠的时间窗口中有 ${count} 个包含成功请求；并非置信区间，也不预测复跑结果。`, + rootTrajectories: '根轨迹数', + kishCoverage: 'Kish 有效根轨迹覆盖数', + kishExplanation: '仅表示覆盖多样性，并非统计有效样本量', + smallestWindow: '最小时间窗口', + successfulRequests: '个成功请求', + }, +} as const; + +const observedWindowDuration = (seconds: number | undefined, locale: Locale): string | null => { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds <= 0) return null; + const minutes = seconds / 60; + if (locale === 'zh') { + return Number.isInteger(minutes) ? `${minutes} 分钟` : `${seconds} 秒`; + } + return Number.isInteger(minutes) ? `${minutes}-minute` : `${seconds}-second`; +}; + +/** + * One-run descriptive diagnostics for the x metric. This intentionally says + * what the range is not: the windows are dependent slices of one run, so the + * min/max cannot support confidence-interval or rerun-prediction language. + */ +const generateObservedWindowRangeHTML = (d: InferenceData, locale: Locale): string => { + const min = d.observedXMin; + const max = d.observedXMax; + const count = d.observed_window_count ?? 0; + if ( + count < 2 || + typeof min !== 'number' || + !Number.isFinite(min) || + min <= 0 || + typeof max !== 'number' || + !Number.isFinite(max) || + max <= min + ) { + return ''; + } + + const t = OBSERVED_RANGE_STRINGS[locale]; + const duration = observedWindowDuration(d.observed_window_seconds, locale); + const title = duration ? t.title(duration) : t.genericTitle; + const expected = d.observed_window_expected_count; + const windowNote = + typeof expected === 'number' && expected > count + ? t.partialWindows(count, expected) + : t.completeWindows(count); + + const coverageRows: string[] = []; + if (typeof d.root_trajectory_count === 'number') { + coverageRows.push(tooltipLine(t.rootTrajectories, d.root_trajectory_count)); + } + if (typeof d.root_trajectory_kish_effective_count === 'number') { + coverageRows.push( + tooltipLine( + t.kishCoverage, + `${fmt(d.root_trajectory_kish_effective_count)} (${t.kishExplanation})`, + ), + ); + } + if (typeof d.observed_window_min_requests === 'number') { + const value = + locale === 'zh' + ? `${d.observed_window_min_requests}${t.successfulRequests}` + : `${d.observed_window_min_requests} ${t.successfulRequests}`; + coverageRows.push(tooltipLine(t.smallestWindow, value)); + } + + return ` +
+
+ ${title}: ${fmt(min)}–${fmt(max)} +
+
+ ${windowNote} +
+ ${coverageRows.join('')} +
`; +}; + const CACHE_STRINGS = { en: { offloadType: 'Offload Type', @@ -313,6 +413,7 @@ export const generateTooltipContent = (config: TooltipConfig): string => { ` : '' } + ${generateObservedWindowRangeHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
@@ -373,6 +474,7 @@ export const generateOverlayTooltipContent = (config: OverlayTooltipConfig): str
${yLabel}: ${fmt(d.y)}
+ ${generateObservedWindowRangeHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
@@ -444,6 +546,7 @@ export const generateGPUGraphTooltipContent = (config: TooltipConfig): string =>
` : '' } + ${generateObservedWindowRangeHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 22a16af3..ce81c909 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -90,6 +90,39 @@ describe('rowToAggDataEntry', () => { expect(entry.median_intvty).toBe(12.5); }); + it('passes AgentX observed-window and root-coverage diagnostics through', () => { + const entry = rowToAggDataEntry( + makeRow({ + metrics: { + observed_window_seconds: 600, + observed_window_expected_count: 6, + observed_window_count: 6, + observed_window_min_requests: 21, + root_trajectory_count: 3, + root_trajectory_kish_effective_count: 1.69, + root_trajectory_largest_share: 0.74, + observed_window_p90_ttft_min: 1.416, + observed_window_p90_ttft_max: 4.114, + observed_window_p75_e2el_min: 42.1, + observed_window_p75_e2el_max: 79.2, + observed_window_p90_intvty_min: 22.6, + observed_window_p90_intvty_max: 30.7, + } as unknown as BenchmarkRow['metrics'], + }), + ); + + expect(entry.observed_window_seconds).toBe(600); + expect(entry.observed_window_count).toBe(6); + expect(entry.observed_window_min_requests).toBe(21); + expect(entry.root_trajectory_count).toBe(3); + expect(entry.root_trajectory_kish_effective_count).toBe(1.69); + expect(entry.root_trajectory_largest_share).toBe(0.74); + expect(entry.observed_window_p90_ttft_min).toBe(1.416); + expect(entry.observed_window_p90_ttft_max).toBe(4.114); + expect(entry.observed_window_p75_e2el_max).toBe(79.2); + expect(entry.observed_window_p90_intvty_min).toBe(22.6); + }); + it('defaults missing metrics to 0', () => { const entry = rowToAggDataEntry(makeRow({ metrics: {} })); expect(entry.tput_per_gpu).toBe(0); diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index 6180b87e..ff0d08e2 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -177,6 +177,25 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { num_requests_successful: m.num_requests_successful, total_prompt_tokens: m.total_prompt_tokens, total_generation_tokens: m.total_generation_tokens, + observed_window_seconds: m.observed_window_seconds, + observed_window_expected_count: m.observed_window_expected_count, + observed_window_count: m.observed_window_count, + observed_window_min_requests: m.observed_window_min_requests, + root_trajectory_count: m.root_trajectory_count, + root_trajectory_kish_effective_count: m.root_trajectory_kish_effective_count, + root_trajectory_largest_share: m.root_trajectory_largest_share, + observed_window_p75_ttft_min: m.observed_window_p75_ttft_min, + observed_window_p75_ttft_max: m.observed_window_p75_ttft_max, + observed_window_p90_ttft_min: m.observed_window_p90_ttft_min, + observed_window_p90_ttft_max: m.observed_window_p90_ttft_max, + observed_window_p75_e2el_min: m.observed_window_p75_e2el_min, + observed_window_p75_e2el_max: m.observed_window_p75_e2el_max, + observed_window_p90_e2el_min: m.observed_window_p90_e2el_min, + observed_window_p90_e2el_max: m.observed_window_p90_e2el_max, + observed_window_p75_intvty_min: m.observed_window_p75_intvty_min, + observed_window_p75_intvty_max: m.observed_window_p75_intvty_max, + observed_window_p90_intvty_min: m.observed_window_p90_intvty_min, + observed_window_p90_intvty_max: m.observed_window_p90_intvty_max, }; } diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 914eed4b..1cab5a38 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -145,4 +145,26 @@ export const METRIC_KEYS = new Set([ 'peak_temp_c', 'avg_util_pct', 'avg_mem_used_mb', + // AgentX realized-workload coverage and non-overlapping time-window ranges. + // Bounds describe one observed run and must never be labeled as confidence + // intervals or rerun-prediction intervals. + 'observed_window_seconds', + 'observed_window_expected_count', + 'observed_window_count', + 'observed_window_min_requests', + 'root_trajectory_count', + 'root_trajectory_kish_effective_count', + 'root_trajectory_largest_share', + 'observed_window_p75_ttft_min', + 'observed_window_p75_ttft_max', + 'observed_window_p90_ttft_min', + 'observed_window_p90_ttft_max', + 'observed_window_p75_e2el_min', + 'observed_window_p75_e2el_max', + 'observed_window_p90_e2el_min', + 'observed_window_p90_e2el_max', + 'observed_window_p75_intvty_min', + 'observed_window_p75_intvty_max', + 'observed_window_p90_intvty_min', + 'observed_window_p90_intvty_max', ]); diff --git a/packages/db/research/agentx-paired-signatures.sql b/packages/db/research/agentx-paired-signatures.sql new file mode 100644 index 00000000..125e4241 --- /dev/null +++ b/packages/db/research/agentx-paired-signatures.sql @@ -0,0 +1,65 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +-- Workload-matched request signatures for two benchmark rows. This is a +-- sensitivity diagnostic, not an uncertainty interval: repeated signatures +-- are generally sparse within one fixed-duration run. +COPY ( +WITH raw AS ( + SELECT + br.id, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS source_trace_id, + COALESCE(NULLIF(r->>'srcOuter', ''), NULLIF(r->>'ti', ''), '0')::int AS source_outer_idx, + COALESCE(NULLIF(r->>'srcInner', ''), '-1')::int AS source_inner_idx, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + ((r->>'end')::float8 - (r->>'start')::float8) / 1e9 AS e2e_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS tpot_s, + NULLIF(r->>'isl', '')::float8 AS isl, + NULLIF(r->>'osl', '')::float8 AS osl + FROM benchmark_results br + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + CROSS JOIN LATERAL jsonb_array_elements(atr.request_timeline->'requests') r + WHERE br.id IN (:id_a, :id_b) + AND r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), signatures AS ( + SELECT + id, + source_trace_id, + source_outer_idx, + source_inner_idx, + count(*)::int AS n, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS median_ttft_s, + percentile_cont(.5) WITHIN GROUP (ORDER BY e2e_s) AS median_e2e_s, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS median_tpot_s, + percentile_cont(.5) WITHIN GROUP (ORDER BY isl) + FILTER (WHERE isl IS NOT NULL) AS median_isl, + percentile_cont(.5) WITHIN GROUP (ORDER BY osl) + FILTER (WHERE osl IS NOT NULL) AS median_osl + FROM raw + GROUP BY id, source_trace_id, source_outer_idx, source_inner_idx +) +SELECT + a.source_trace_id, + a.source_outer_idx, + a.source_inner_idx, + a.n AS n_a, + b.n AS n_b, + a.median_ttft_s AS ttft_a, + b.median_ttft_s AS ttft_b, + a.median_e2e_s AS e2e_a, + b.median_e2e_s AS e2e_b, + a.median_tpot_s AS tpot_a, + b.median_tpot_s AS tpot_b, + a.median_isl AS isl_a, + b.median_isl AS isl_b, + a.median_osl AS osl_a, + b.median_osl AS osl_b +FROM signatures a +JOIN signatures b USING (source_trace_id, source_outer_idx, source_inner_idx) +WHERE a.id = :id_a + AND b.id = :id_b +ORDER BY a.source_trace_id, a.source_outer_idx, a.source_inner_idx +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/research/agentx-trace-coverage.sql b/packages/db/research/agentx-trace-coverage.sql new file mode 100644 index 00000000..f6ed4f19 --- /dev/null +++ b/packages/db/research/agentx-trace-coverage.sql @@ -0,0 +1,113 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +-- Root-trajectory exposure for overlap and repeated-configuration analyses. +COPY ( +WITH canonical AS ( + SELECT + br.id, + br.workflow_run_id, + br.date, + br.conc, + br.image, + br.offload_mode, + br.config_id, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + c.disagg, + c.is_multinode, + c.prefill_tp, + c.prefill_ep, + c.prefill_dp_attention, + c.prefill_num_workers, + c.decode_tp, + c.decode_ep, + c.decode_dp_attention, + c.decode_num_workers, + c.num_prefill_gpu, + c.num_decode_gpu, + COALESCE(ds.dataset_slugs, '') AS dataset_slugs, + atr.request_timeline + FROM benchmark_results br + JOIN configs c ON c.id = br.config_id + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + LEFT JOIN LATERAL ( + SELECT string_agg(rd.dataset_slug, ',' ORDER BY rd.dataset_slug) AS dataset_slugs + FROM run_datasets rd + WHERE rd.workflow_run_id = br.workflow_run_id + ) ds ON true + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND (br.metrics->>'duration_seconds')::float8 BETWEEN 3500 AND 3700 + AND mod(br.id, 10) = :shard + AND br.id <= :max_id +), trace_counts AS ( + SELECT + c.id, + c.workflow_run_id, + c.date, + c.conc, + c.image, + c.offload_mode, + c.config_id, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + c.disagg, + c.is_multinode, + c.prefill_tp, + c.prefill_ep, + c.prefill_dp_attention, + c.prefill_num_workers, + c.decode_tp, + c.decode_ep, + c.decode_dp_attention, + c.decode_num_workers, + c.num_prefill_gpu, + c.num_decode_gpu, + c.dataset_slugs, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS source_trace_id, + count(*)::int AS n_requests, + min((r->>'credit')::float8) AS first_credit_ns, + max((r->>'credit')::float8) AS last_credit_ns + FROM canonical c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false + GROUP BY + c.id, + c.workflow_run_id, + c.date, + c.conc, + c.image, + c.offload_mode, + c.config_id, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + c.disagg, + c.is_multinode, + c.prefill_tp, + c.prefill_ep, + c.prefill_dp_attention, + c.prefill_num_workers, + c.decode_tp, + c.decode_ep, + c.decode_dp_attention, + c.decode_num_workers, + c.num_prefill_gpu, + c.num_decode_gpu, + c.dataset_slugs, + source_trace_id +) +SELECT * +FROM trace_counts +ORDER BY id, first_credit_ns, source_trace_id +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/research/agentx-uncertainty-diagnostics.sql b/packages/db/research/agentx-uncertainty-diagnostics.sql new file mode 100644 index 00000000..e96c88db --- /dev/null +++ b/packages/db/research/agentx-uncertainty-diagnostics.sql @@ -0,0 +1,228 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +COPY ( +WITH canonical AS ( + SELECT + br.id, + br.conc, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + br.offload_mode, + atr.request_timeline + FROM benchmark_results br + JOIN configs c ON c.id = br.config_id + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND (br.metrics->>'duration_seconds')::float8 BETWEEN 3500 AND 3700 + -- Run with `psql -v shard=0` through `-v shard=9`. Keeping each read + -- replica snapshot short avoids cancellation during Neon recovery. + AND mod(br.id, 10) = :shard + -- Pin this to the maximum id observed before launching the ten shards so + -- a concurrent ingest cannot make the local analysis internally mixed. + AND br.id <= :max_id +), raw AS ( + SELECT + c.id, + c.conc, + c.hardware, + c.framework, + c.model, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS cluster_id, + concat_ws( + '|', + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown'), + COALESCE(NULLIF(r->>'srcOuter', ''), NULLIF(r->>'ti', ''), '0'), + COALESCE(NULLIF(r->>'srcInner', ''), '-1') + ) AS request_signature, + NULLIF(r->>'cid', '') AS cid, + NULLIF(r->>'wid', '') AS wid, + (r->>'credit')::float8 AS credit_ns, + (r->>'start')::float8 AS start_ns, + (r->>'end')::float8 AS end_ns, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS tpot_s, + NULLIF(r->>'isl', '')::float8 AS isl, + NULLIF(r->>'osl', '')::float8 AS osl + FROM canonical c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), requests AS ( + SELECT *, min(credit_ns) OVER (PARTITION BY id) AS origin_ns + FROM raw +), cluster_counts AS ( + SELECT id, cluster_id, count(*) AS n + FROM requests + GROUP BY id, cluster_id +), cluster_summary AS ( + SELECT + id, + sum(n)::int AS n_requests, + count(*)::int AS source_trajectories, + (sum(n)::float8 * sum(n)) / sum(n::float8 * n) AS kish_source_trajectories, + max(n)::float8 / sum(n) AS largest_source_share + FROM cluster_counts + GROUP BY id +), other_counts AS ( + SELECT + id, + count(DISTINCT cid)::int AS replay_trajectories, + count(DISTINCT wid)::int AS workers + FROM requests + GROUP BY id +), point_values AS ( + SELECT + id, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) AS e2e_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns < 1800e9) AS ttft_first_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns >= 1800e9) AS ttft_second_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns < 1800e9) AS e2e_first_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns >= 1800e9) AS e2e_second_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns < 1800e9) AS tpot_first_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns >= 1800e9) AS tpot_second_half + FROM requests + GROUP BY id +), block_values AS ( + SELECT + id, + floor((credit_ns - origin_ns) / 600e9)::int AS block, + count(*)::int AS n, + count(DISTINCT cluster_id)::int AS source_trajectories, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) AS e2e_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p90 + FROM requests + WHERE floor((credit_ns - origin_ns) / 600e9) BETWEEN 0 AND 5 + GROUP BY id, block +), block_summary AS ( + SELECT + id, + count(*)::int AS n_blocks, + min(n)::int AS min_block_n, + min(source_trajectories)::int AS min_block_trajectories, + min(ttft_p75) AS ttft_p75_min, + max(ttft_p75) AS ttft_p75_max, + min(ttft_p90) AS ttft_p90_min, + max(ttft_p90) AS ttft_p90_max, + min(e2e_p90) AS e2e_p90_min, + max(e2e_p90) AS e2e_p90_max, + min(tpot_p90) AS tpot_p90_min, + max(tpot_p90) AS tpot_p90_max + FROM block_values + GROUP BY id +), signature_halves AS ( + SELECT + id, + cluster_id, + request_signature, + count(*) FILTER (WHERE credit_ns - origin_ns < 1800e9)::int AS first_half_n, + count(*) FILTER (WHERE credit_ns - origin_ns >= 1800e9)::int AS second_half_n, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns < 1800e9) AS ttft_first_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns >= 1800e9) AS ttft_second_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns < 1800e9) AS e2e_first_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns >= 1800e9) AS e2e_second_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns < 1800e9) AS tpot_first_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns >= 1800e9) AS tpot_second_half + FROM requests + GROUP BY id, cluster_id, request_signature +), paired_signature_summary AS ( + SELECT + id, + count(*) FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND ttft_first_half > 0 AND ttft_second_half > 0 + )::int AS paired_signatures, + count(DISTINCT cluster_id) FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND ttft_first_half > 0 AND ttft_second_half > 0 + )::int AS paired_source_trajectories, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_second_half / ttft_first_half) + FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND ttft_first_half > 0 AND ttft_second_half > 0 + ) AS matched_ttft_half_ratio, + percentile_cont(.5) WITHIN GROUP (ORDER BY e2e_second_half / e2e_first_half) + FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND e2e_first_half > 0 AND e2e_second_half > 0 + ) AS matched_e2e_half_ratio, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_second_half / tpot_first_half) + FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND tpot_first_half > 0 AND tpot_second_half > 0 + ) AS matched_tpot_half_ratio + FROM signature_halves + GROUP BY id +) +SELECT + c.id, + c.conc, + c.hardware, + c.framework, + c.model, + cs.n_requests, + cs.source_trajectories, + cs.kish_source_trajectories, + cs.largest_source_share, + oc.replay_trajectories, + oc.workers, + pv.ttft_p75, + pv.ttft_p90, + pv.e2e_p90, + CASE WHEN pv.tpot_p90 > 0 THEN 1 / pv.tpot_p90 END AS interactivity_p90, + bs.n_blocks, + bs.min_block_n, + bs.min_block_trajectories, + bs.ttft_p75_min, + bs.ttft_p75_max, + bs.ttft_p90_min, + bs.ttft_p90_max, + bs.e2e_p90_min, + bs.e2e_p90_max, + CASE WHEN bs.tpot_p90_max > 0 THEN 1 / bs.tpot_p90_max END AS interactivity_p90_min, + CASE WHEN bs.tpot_p90_min > 0 THEN 1 / bs.tpot_p90_min END AS interactivity_p90_max, + bs.ttft_p90_max / NULLIF(bs.ttft_p90_min, 0) AS ttft_block_ratio, + bs.e2e_p90_max / NULLIF(bs.e2e_p90_min, 0) AS e2e_block_ratio, + bs.tpot_p90_max / NULLIF(bs.tpot_p90_min, 0) AS tpot_block_ratio, + pv.ttft_second_half / NULLIF(pv.ttft_first_half, 0) AS ttft_half_ratio, + pv.e2e_second_half / NULLIF(pv.e2e_first_half, 0) AS e2e_half_ratio, + pv.tpot_second_half / NULLIF(pv.tpot_first_half, 0) AS tpot_half_ratio + ,ps.paired_signatures + ,ps.paired_source_trajectories + ,ps.matched_ttft_half_ratio + ,ps.matched_e2e_half_ratio + ,ps.matched_tpot_half_ratio +FROM canonical c +JOIN cluster_summary cs USING (id) +JOIN other_counts oc USING (id) +JOIN point_values pv USING (id) +JOIN block_summary bs USING (id) +JOIN paired_signature_summary ps USING (id) +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/research/agentx-window-sensitivity.sql b/packages/db/research/agentx-window-sensitivity.sql new file mode 100644 index 00000000..c5383859 --- /dev/null +++ b/packages/db/research/agentx-window-sensitivity.sql @@ -0,0 +1,127 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +-- Non-overlapping, equal-duration slices for the pinned one-hour AgentX cohort. +-- Run with psql variables `shard=0..9` and `max_id=`. +-- Equal-duration slices avoid treating a short final wall-clock bucket as if it +-- had the same exposure as a complete bucket. +COPY ( +WITH canonical AS ( + SELECT + br.id, + br.conc, + c.hardware, + c.framework, + c.model, + atr.request_timeline + FROM benchmark_results br + JOIN configs c ON c.id = br.config_id + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND (br.metrics->>'duration_seconds')::float8 BETWEEN 3500 AND 3700 + AND mod(br.id, 10) = :shard + AND br.id <= :max_id +), raw AS ( + SELECT + c.id, + c.conc, + c.hardware, + c.framework, + c.model, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS cluster_id, + (r->>'credit')::float8 AS credit_ns, + (r->>'end')::float8 AS end_ns, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + ((r->>'end')::float8 - (r->>'start')::float8) / 1e9 AS e2e_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS tpot_s, + NULLIF(r->>'isl', '')::float8 AS isl, + NULLIF(r->>'osl', '')::float8 AS osl + FROM canonical c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), timed AS ( + SELECT + *, + min(credit_ns) OVER (PARTITION BY id) AS origin_ns, + max(credit_ns) OVER (PARTITION BY id) - min(credit_ns) OVER (PARTITION BY id) AS span_ns + FROM raw +), expanded AS ( + SELECT + t.*, + w.window_minutes, + w.n_blocks, + least( + w.n_blocks - 1, + floor((t.credit_ns - t.origin_ns) / NULLIF(t.span_ns, 0) * w.n_blocks)::int + ) AS block + FROM timed t + CROSS JOIN ( + VALUES + (5, 12), + (10, 6), + (15, 4), + (20, 3) + ) AS w(window_minutes, n_blocks) +), cluster_counts AS ( + SELECT id, window_minutes, block, cluster_id, count(*) AS n + FROM expanded + GROUP BY id, window_minutes, block, cluster_id +), cluster_summary AS ( + SELECT + id, + window_minutes, + block, + count(*)::int AS source_trajectories, + (sum(n)::float8 * sum(n)) / NULLIF(sum(n::float8 * n), 0) AS kish_source_trajectories, + max(n)::float8 / NULLIF(sum(n), 0) AS largest_source_share + FROM cluster_counts + GROUP BY id, window_minutes, block +), block_values AS ( + SELECT + id, + conc, + hardware, + framework, + model, + window_minutes, + n_blocks, + block, + count(*)::int AS n_requests, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p90, + percentile_cont(.75) WITHIN GROUP (ORDER BY e2e_s) AS e2e_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY e2e_s) AS e2e_p90, + percentile_cont(.75) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p90, + avg(isl) AS isl_mean, + percentile_cont(.5) WITHIN GROUP (ORDER BY isl) + FILTER (WHERE isl IS NOT NULL) AS isl_p50, + percentile_cont(.9) WITHIN GROUP (ORDER BY isl) + FILTER (WHERE isl IS NOT NULL) AS isl_p90, + avg(osl) AS osl_mean, + percentile_cont(.5) WITHIN GROUP (ORDER BY osl) + FILTER (WHERE osl IS NOT NULL) AS osl_p50, + percentile_cont(.9) WITHIN GROUP (ORDER BY osl) + FILTER (WHERE osl IS NOT NULL) AS osl_p90, + sum(COALESCE(isl, 0) + COALESCE(osl, 0)) / (window_minutes * 60) AS token_rate + FROM expanded + WHERE block BETWEEN 0 AND n_blocks - 1 + GROUP BY id, conc, hardware, framework, model, window_minutes, n_blocks, block +) +SELECT + b.*, + c.source_trajectories, + c.kish_source_trajectories, + c.largest_source_share, + CASE WHEN b.tpot_p75 > 0 THEN 1 / b.tpot_p75 END AS interactivity_p75, + CASE WHEN b.tpot_p90 > 0 THEN 1 / b.tpot_p90 END AS interactivity_p90 +FROM block_values b +JOIN cluster_summary c USING (id, window_minutes, block) +ORDER BY id, window_minutes, block +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/src/etl/agentic-v3-flatten.ts b/packages/db/src/etl/agentic-v3-flatten.ts index a3c223af..80656094 100644 --- a/packages/db/src/etl/agentic-v3-flatten.ts +++ b/packages/db/src/etl/agentic-v3-flatten.ts @@ -63,6 +63,21 @@ const V3_SCALAR_PATHS: [string[], string][] = [ [['server_metrics', 'tokens', 'prompt_total'], 'total_prompt_tokens'], [['server_metrics', 'tokens', 'generation_total'], 'total_generation_tokens'], [['server_metrics', 'tokens', 'requests_completed'], 'total_requests_completed'], + // One-run AgentX coverage and non-overlapping-window diagnostics. These + // describe the realized hour; they are not confidence intervals. + [['request_metrics', 'stability', 'window_seconds'], 'observed_window_seconds'], + [['request_metrics', 'stability', 'expected_window_count'], 'observed_window_expected_count'], + [['request_metrics', 'stability', 'observed_window_count'], 'observed_window_count'], + [['request_metrics', 'stability', 'min_window_requests'], 'observed_window_min_requests'], + [['request_metrics', 'stability', 'root_trajectory_count'], 'root_trajectory_count'], + [ + ['request_metrics', 'stability', 'root_trajectory_kish_effective_count'], + 'root_trajectory_kish_effective_count', + ], + [ + ['request_metrics', 'stability', 'root_trajectory_largest_share'], + 'root_trajectory_largest_share', + ], // Deliberately NOT mapped (yet): cache.overall/prefix_cache_hits/queries, // kv_cache.cpu_*, tokens.prompt_by_source, sources[] — new v3 detail we don't // consume anywhere; add here + METRIC_KEYS when a view needs them. @@ -127,5 +142,25 @@ export function flattenAgenticAggRow(row: Record): Record = {}): Record { expect(m.total_prompt_tokens).toBe(261750519); expect(m.total_generation_tokens).toBe(1422696); expect(m.total_requests_completed).toBe(1648); + // realized coverage + non-overlapping-window diagnostics + expect(m.observed_window_seconds).toBe(600); + expect(m.observed_window_count).toBe(6); + expect(m.observed_window_min_requests).toBe(190); + expect(m.root_trajectory_count).toBe(25); + expect(m.root_trajectory_kish_effective_count).toBeCloseTo(14.767, 3); + expect(m.observed_window_p90_ttft_min).toBeCloseTo(1.416, 3); + expect(m.observed_window_p90_ttft_max).toBeCloseTo(4.114, 3); + expect(m.observed_window_p75_e2el_max).toBeCloseTo(79.2, 3); + expect(m.observed_window_p90_intvty_min).toBeCloseTo(22.621, 3); // nested containers must not leak into metrics expect(m).not.toHaveProperty('request_metrics'); expect(m).not.toHaveProperty('server_metrics'); From d2428614e8ea8647c9f116a1680f4178c1b0f3b1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 21 Jul 2026 15:42:09 -0500 Subject: [PATCH 2/3] add AgentX stability backfill --- .../db/research/agentx-stability-backfill.sql | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 packages/db/research/agentx-stability-backfill.sql diff --git a/packages/db/research/agentx-stability-backfill.sql b/packages/db/research/agentx-stability-backfill.sql new file mode 100644 index 00000000..4bebfba2 --- /dev/null +++ b/packages/db/research/agentx-stability-backfill.sql @@ -0,0 +1,190 @@ +\set ON_ERROR_STOP on + +-- Idempotently backfill descriptive AgentX stability fields from the stored +-- request timeline. These are observed non-overlapping-window ranges, not +-- confidence intervals or rerun prediction intervals. +-- +-- Required psql variables: +-- expected_branch_id Neon branch id this write is allowed to target +-- Optional psql variables: +-- min_id inclusive benchmark_results.id (default 0) +-- max_id inclusive benchmark_results.id (default bigint max) + +\if :{?expected_branch_id} +\else + \echo 'Refusing to run: expected_branch_id is required.' + \quit 3 +\endif + +\if :{?min_id} +\else + \set min_id 0 +\endif + +\if :{?max_id} +\else + \set max_id 9223372036854775807 +\endif + +SELECT current_setting('neon.branch_id', true) = :'expected_branch_id' AS on_expected_branch +\gset + +\if :on_expected_branch + \echo 'Verified Neon branch' :expected_branch_id +\else + \echo 'Refusing to run: connected Neon branch does not match' :expected_branch_id + \quit 3 +\endif + +BEGIN; +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '15min'; + +CREATE TEMP TABLE _agentx_stability_patch ON COMMIT DROP AS +WITH candidates AS ( + SELECT + br.id, + floor((br.metrics->>'duration_seconds')::float8 / 600)::int AS expected_windows, + atr.request_timeline + FROM benchmark_results br + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND br.id BETWEEN :min_id::bigint AND :max_id::bigint + AND (br.metrics->>'duration_seconds')::float8 >= 1200 + AND jsonb_typeof(atr.request_timeline->'requests') = 'array' +), raw AS ( + SELECT + c.id, + c.expected_windows, + COALESCE( + NULLIF(r->>'credit', '')::float8, + NULLIF(r->>'start', '')::float8 + ) AS timestamp_ns, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS root_id, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + CASE + WHEN NULLIF(r->>'end', '')::float8 > NULLIF(r->>'start', '')::float8 + THEN (NULLIF(r->>'end', '')::float8 - NULLIF(r->>'start', '')::float8) / 1e9 + END AS e2el_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS itl_s + FROM candidates c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE COALESCE(NULLIF(r->>'phase', ''), 'profiling') = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), timed AS ( + SELECT + *, + min(timestamp_ns) OVER (PARTITION BY id) AS origin_ns + FROM raw + WHERE timestamp_ns IS NOT NULL +), windowed AS ( + SELECT + *, + floor((timestamp_ns - origin_ns) / 600e9)::int AS window_index + FROM timed +), window_metrics AS ( + SELECT + id, + expected_windows, + window_index, + count(*)::int AS request_count, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s > 0) AS p75_ttft, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s > 0) AS p90_ttft, + percentile_cont(.75) WITHIN GROUP (ORDER BY e2el_s) + FILTER (WHERE e2el_s > 0) AS p75_e2el, + percentile_cont(.9) WITHIN GROUP (ORDER BY e2el_s) + FILTER (WHERE e2el_s > 0) AS p90_e2el, + percentile_cont(.75) WITHIN GROUP (ORDER BY itl_s) + FILTER (WHERE itl_s > 0) AS p75_itl, + percentile_cont(.9) WITHIN GROUP (ORDER BY itl_s) + FILTER (WHERE itl_s > 0) AS p90_itl + FROM windowed + WHERE window_index >= 0 + AND window_index < expected_windows + GROUP BY id, expected_windows, window_index +), window_summary AS ( + SELECT + id, + max(expected_windows)::int AS expected_window_count, + count(*)::int AS observed_window_count, + min(request_count)::int AS min_window_requests, + min(p75_ttft) AS p75_ttft_min, + max(p75_ttft) AS p75_ttft_max, + min(p90_ttft) AS p90_ttft_min, + max(p90_ttft) AS p90_ttft_max, + min(p75_e2el) AS p75_e2el_min, + max(p75_e2el) AS p75_e2el_max, + min(p90_e2el) AS p90_e2el_min, + max(p90_e2el) AS p90_e2el_max, + CASE WHEN max(p75_itl) > 0 THEN 1 / max(p75_itl) END AS p75_intvty_min, + CASE WHEN min(p75_itl) > 0 THEN 1 / min(p75_itl) END AS p75_intvty_max, + CASE WHEN max(p90_itl) > 0 THEN 1 / max(p90_itl) END AS p90_intvty_min, + CASE WHEN min(p90_itl) > 0 THEN 1 / min(p90_itl) END AS p90_intvty_max + FROM window_metrics + GROUP BY id +), root_counts AS ( + SELECT id, root_id, count(*)::bigint AS request_count + FROM raw + GROUP BY id, root_id +), root_summary AS ( + SELECT + id, + count(*)::int AS root_trajectory_count, + (sum(request_count)::float8 * sum(request_count)) + / NULLIF(sum(request_count::float8 * request_count), 0) + AS root_trajectory_kish_effective_count, + max(request_count)::float8 / NULLIF(sum(request_count), 0) + AS root_trajectory_largest_share + FROM root_counts + GROUP BY id +), patches AS ( + SELECT + ws.id, + jsonb_strip_nulls(jsonb_build_object( + 'observed_window_seconds', 600, + 'observed_window_expected_count', ws.expected_window_count, + 'observed_window_count', ws.observed_window_count, + 'observed_window_min_requests', ws.min_window_requests, + 'root_trajectory_count', rs.root_trajectory_count, + 'root_trajectory_kish_effective_count', + round(rs.root_trajectory_kish_effective_count::numeric, 5), + 'root_trajectory_largest_share', + round(rs.root_trajectory_largest_share::numeric, 5), + 'observed_window_p75_ttft_min', round(ws.p75_ttft_min::numeric, 5), + 'observed_window_p75_ttft_max', round(ws.p75_ttft_max::numeric, 5), + 'observed_window_p90_ttft_min', round(ws.p90_ttft_min::numeric, 5), + 'observed_window_p90_ttft_max', round(ws.p90_ttft_max::numeric, 5), + 'observed_window_p75_e2el_min', round(ws.p75_e2el_min::numeric, 5), + 'observed_window_p75_e2el_max', round(ws.p75_e2el_max::numeric, 5), + 'observed_window_p90_e2el_min', round(ws.p90_e2el_min::numeric, 5), + 'observed_window_p90_e2el_max', round(ws.p90_e2el_max::numeric, 5), + 'observed_window_p75_intvty_min', round(ws.p75_intvty_min::numeric, 5), + 'observed_window_p75_intvty_max', round(ws.p75_intvty_max::numeric, 5), + 'observed_window_p90_intvty_min', round(ws.p90_intvty_min::numeric, 5), + 'observed_window_p90_intvty_max', round(ws.p90_intvty_max::numeric, 5) + )) AS patch + FROM window_summary ws + JOIN root_summary rs USING (id) +) +SELECT id, patch +FROM patches; + +WITH updated AS ( + UPDATE benchmark_results br + SET metrics = br.metrics || p.patch + FROM _agentx_stability_patch p + WHERE br.id = p.id + AND br.metrics IS DISTINCT FROM br.metrics || p.patch + RETURNING br.id +) +SELECT + (SELECT count(*) FROM _agentx_stability_patch) AS computed_rows, + count(*) AS updated_rows, + min(id) AS first_updated_id, + max(id) AS last_updated_id +FROM updated; + +COMMIT; From 52229d55acfed00adbf77d59a0fec3b8ee78b86d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 21 Jul 2026 17:21:03 -0500 Subject: [PATCH 3/3] feat(agentic): show cumulative convergence whiskers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backfill and ingest retrospective cumulative-prefix stabilization diagnostics, then render post-stabilization Pareto whiskers for official and unofficial AgentX runs with bilingual explanatory tooltips. 中文：回填并摄取回溯性累计前缀收敛诊断，在官方与非官方 AgentX 运行的 Pareto 前沿上展示稳定后的须线，并提供中英双语说明。 --- .../cypress/component/scatter-graph.cy.tsx | 22 ++- .../app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 98 ++++++++-- .../inference/hooks/useChartData.ts | 4 +- .../app/src/components/inference/types.ts | 49 ++++- .../ui/ScatterGraph.decoration.test.tsx | 51 ++++-- .../components/inference/ui/ScatterGraph.tsx | 173 +++++++++--------- .../src/components/inference/utils.test.ts | 22 ++- .../app/src/components/inference/utils.ts | 4 +- .../inference/utils/convergence-range.test.ts | 114 ++++++++++++ .../inference/utils/convergence-range.ts | 78 ++++++++ .../utils/observed-window-range.test.ts | 90 --------- .../inference/utils/observed-window-range.ts | 41 ----- .../inference/utils/tooltip-utils.test.ts | 76 +++++--- .../inference/utils/tooltipUtils.ts | 133 ++++++++------ .../app/src/lib/benchmark-transform.test.ts | 20 +- packages/app/src/lib/benchmark-transform.ts | 34 ++++ packages/constants/src/metric-keys.ts | 37 ++++ .../db/research/agentx-stability-backfill.sql | 167 ++++++++++++++++- packages/db/src/etl/agentic-v3-flatten.ts | 37 ++++ packages/db/src/etl/benchmark-mapper.test.ts | 47 +++++ 20 files changed, 929 insertions(+), 368 deletions(-) create mode 100644 packages/app/src/components/inference/utils/convergence-range.test.ts create mode 100644 packages/app/src/components/inference/utils/convergence-range.ts delete mode 100644 packages/app/src/components/inference/utils/observed-window-range.test.ts delete mode 100644 packages/app/src/components/inference/utils/observed-window-range.ts diff --git a/packages/app/cypress/component/scatter-graph.cy.tsx b/packages/app/cypress/component/scatter-graph.cy.tsx index dba15bfc..4772d79e 100644 --- a/packages/app/cypress/component/scatter-graph.cy.tsx +++ b/packages/app/cypress/component/scatter-graph.cy.tsx @@ -732,10 +732,12 @@ describe('ScatterGraph', () => { y: 260 - index * 40, precision: Precision.FP4, run_url: runUrl, - observedXMin: x * 0.75, - observedXMax: x * 1.25, - observed_window_seconds: 600, - observed_window_count: 6, + convergenceXMin: x * 0.96, + convergenceXMax: x * 1.04, + convergenceTimeSeconds: 1200, + convergenceRequests: 407, + convergenceMaxRelativeDeviation: 0.04, + convergence_tolerance_ratio: 0.05, }), ), hardwareConfig: hwConfig, @@ -781,16 +783,16 @@ describe('ScatterGraph', () => { cy.get('#test-scatter-dismiss-preview svg .unofficial-overlay-pt').should('have.length', 3); cy.get('#test-scatter-dismiss-preview svg .overlay-roofline-path').should('exist'); - cy.get( - '#test-scatter-dismiss-preview svg .observed-window-range[data-source="overlay"]', - ).should('exist'); + cy.get('#test-scatter-dismiss-preview svg .convergence-range[data-source="overlay"]').should( + 'exist', + ); cy.get('[data-testid="dismiss-preview"]').click(); cy.get('#test-scatter-dismiss-preview svg .unofficial-overlay-pt').should('not.exist'); cy.get('#test-scatter-dismiss-preview svg .overlay-roofline-path').should('not.exist'); - cy.get( - '#test-scatter-dismiss-preview svg .observed-window-range[data-source="overlay"]', - ).should('not.exist'); + cy.get('#test-scatter-dismiss-preview svg .convergence-range[data-source="overlay"]').should( + 'not.exist', + ); }); }); diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index ecd5f04a..c3b6b8f4 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -41,7 +41,7 @@ const percentileLadder = (prefix: string, base: number): Record [`std_${prefix}`]: base * 0.3, }); -const agenticMetrics = (conc: number): Record => { +const agenticMetrics = (conc: number, stabilized = true): Record => { const scale = conc / 16; const itl = 0.011 * scale; const p75Ttft = 0.4 * scale * 1.2; @@ -83,6 +83,44 @@ const agenticMetrics = (conc: number): Record => { observed_window_p75_intvty_max: p75Interactivity * 1.28, observed_window_p90_intvty_min: p90Interactivity * 0.68, observed_window_p90_intvty_max: p90Interactivity * 1.32, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + ...(stabilized + ? { + convergence_p75_ttft_time_seconds: 900, + convergence_p75_ttft_requests: Math.round(conc * 25), + convergence_p75_ttft_min: p75Ttft * 0.97, + convergence_p75_ttft_max: p75Ttft * 1.03, + convergence_p75_ttft_max_relative_deviation: 0.03, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: Math.round(conc * 25.4), + convergence_p90_ttft_min: p90Ttft * 0.96, + convergence_p90_ttft_max: p90Ttft * 1.04, + convergence_p90_ttft_max_relative_deviation: 0.04, + convergence_p75_e2el_time_seconds: 900, + convergence_p75_e2el_requests: Math.round(conc * 25), + convergence_p75_e2el_min: p75E2e * 0.97, + convergence_p75_e2el_max: p75E2e * 1.03, + convergence_p75_e2el_max_relative_deviation: 0.03, + convergence_p90_e2el_time_seconds: 1200, + convergence_p90_e2el_requests: Math.round(conc * 25.4), + convergence_p90_e2el_min: p90E2e * 0.96, + convergence_p90_e2el_max: p90E2e * 1.04, + convergence_p90_e2el_max_relative_deviation: 0.04, + convergence_p75_intvty_time_seconds: 900, + convergence_p75_intvty_requests: Math.round(conc * 25), + convergence_p75_intvty_min: p75Interactivity * 0.97, + convergence_p75_intvty_max: p75Interactivity * 1.03, + convergence_p75_intvty_max_relative_deviation: 0.03, + convergence_p90_intvty_time_seconds: 1200, + convergence_p90_intvty_requests: Math.round(conc * 25.4), + convergence_p90_intvty_min: p90Interactivity * 0.96, + convergence_p90_intvty_max: p90Interactivity * 1.04, + convergence_p90_intvty_max_relative_deviation: 0.04, + } + : {}), }; }; @@ -141,7 +179,7 @@ const agenticBenchmarks = agenticGpus.flatMap((g) => offload_mode: 'off', benchmark_type: 'agentic_traces', image: 'vllm/vllm-openai:v0.9.0', - metrics: agenticMetrics(conc), + metrics: agenticMetrics(conc, !(g.hardware === 'b200' && conc === 16)), workers: null, date: AGENTIC_DATE, run_url: null, @@ -225,26 +263,60 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); }); - it('renders observed-window ranges with an explicit non-inferential caveat', () => { + it('renders post-stabilization spans with an explicit non-inferential caveat', () => { cy.get( - '[data-testid="inference-chart-display"] [data-testid="observed-window-range"][data-source="official"]', + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="official"]', ).should('have.length.greaterThan', 0); - cy.get('[data-testid="observed-window-range-key"]') - .should('contain.text', 'Observed 10-minute range') + cy.get('[data-testid="convergence-range-key"]') + .should('contain.text', 'Cumulative span after ±5% stabilization') + .and('contain.text', 'retrospective') .and('contain.text', 'not a confidence interval'); - cy.get('[data-testid="scatter-graph"] svg .dot-group .visible-shape') - .first() - .click({ force: true }); + cy.get('[data-testid="scatter-graph"] svg .dot-group').then(($dots) => { + const stabilizedDot = [...$dots].find( + (dot) => + (dot as unknown as SVGGElement & { __data__: { id?: number } }).__data__.id === 900001, + ); + expect(stabilizedDot, 'stabilized fixture point').not.to.equal(undefined); + cy.wrap(stabilizedDot).find('.visible-shape').click({ force: true }); + }); cy.get('[data-chart-tooltip]:visible') - .should('contain.text', 'Observed 10-minute range') + .should('contain.text', 'Stabilized by 20 minutes at ±5%') + .and('contain.text', 'Post-stabilization span') + .and('contain.text', 'Maximum later deviation') + .and('contain.text', 'Retrospective within-run diagnostic') .and('contain.text', 'not a confidence interval or rerun prediction') .and('contain.text', 'Kish-effective root coverage'); }); - it('keeps observed-window ranges anchored through chart zoom', () => { + it('renders no whisker when the cumulative estimate did not stabilize', () => { + cy.get( + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="official"]', + ).then(($ranges) => { + const hasUnstablePoint = [...$ranges].some( + (range) => + (range as unknown as SVGGElement & { __data__: { point: { id?: number } } }).__data__ + .point.id === 900000, + ); + expect(hasUnstablePoint).to.equal(false); + }); + + cy.get('[data-testid="scatter-graph"] svg .dot-group').then(($dots) => { + const unstableDot = [...$dots].find( + (dot) => + (dot as unknown as SVGGElement & { __data__: { id?: number } }).__data__.id === 900000, + ); + expect(unstableDot, 'non-stabilized fixture point').not.to.equal(undefined); + cy.wrap(unstableDot).find('.visible-shape').click({ force: true }); + }); + cy.get('[data-chart-tooltip]:visible') + .should('contain.text', 'Not stabilized within the 60-minute run at ±5%') + .and('not.contain.text', 'Post-stabilization span'); + }); + + it('keeps convergence ranges anchored through chart zoom', () => { cy.get( - '[data-testid="inference-chart-display"] [data-testid="observed-window-range"][data-source="official"] .observed-window-range-stem', + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="official"] .convergence-range-stem', ) .first() .then(($stem) => { @@ -456,7 +528,7 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () expect(total).to.be.greaterThan(0); }); cy.get( - '[data-testid="inference-chart-display"] [data-testid="observed-window-range"][data-source="overlay"]', + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="overlay"]', ) .should('have.length.greaterThan', 0) .and('have.attr', 'data-run-index', '0'); diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index 7b4b897b..31732a7a 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -29,7 +29,7 @@ import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { calculateCostsForGpus, calculatePowerForGpus } from '@/lib/utils'; import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; -import { withObservedWindowRange } from '@/components/inference/utils/observed-window-range'; +import { withConvergenceRange } from '@/components/inference/utils/convergence-range'; import { applyQuickFilters, computeAvailableQuickFilters, @@ -154,7 +154,7 @@ export function remapChartPoint( const metric = point[metricKey] as { y: number; roof: boolean } | undefined; const xCandidate = (point as Partial)[xAxisField]; - return withObservedWindowRange( + return withConvergenceRange( { ...point, x: typeof xCandidate === 'number' ? xCandidate : point.x, diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index f7cd530a..f1f6d2d3 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -245,6 +245,44 @@ export interface AggDataEntry { observed_window_p75_intvty_max?: number; observed_window_p90_intvty_min?: number; observed_window_p90_intvty_max?: number; + /** Width of each cumulative convergence checkpoint, in seconds. */ + convergence_checkpoint_seconds?: number; + /** Symmetric multiplicative tolerance around the final cumulative estimate. */ + convergence_tolerance_ratio?: number; + /** Required duration after stabilization, in seconds. */ + convergence_min_confirmation_seconds?: number; + /** Last complete cumulative checkpoint included in the diagnostic. */ + convergence_horizon_seconds?: number; + convergence_p75_ttft_time_seconds?: number; + convergence_p75_ttft_requests?: number; + convergence_p75_ttft_min?: number; + convergence_p75_ttft_max?: number; + convergence_p75_ttft_max_relative_deviation?: number; + convergence_p90_ttft_time_seconds?: number; + convergence_p90_ttft_requests?: number; + convergence_p90_ttft_min?: number; + convergence_p90_ttft_max?: number; + convergence_p90_ttft_max_relative_deviation?: number; + convergence_p75_e2el_time_seconds?: number; + convergence_p75_e2el_requests?: number; + convergence_p75_e2el_min?: number; + convergence_p75_e2el_max?: number; + convergence_p75_e2el_max_relative_deviation?: number; + convergence_p90_e2el_time_seconds?: number; + convergence_p90_e2el_requests?: number; + convergence_p90_e2el_min?: number; + convergence_p90_e2el_max?: number; + convergence_p90_e2el_max_relative_deviation?: number; + convergence_p75_intvty_time_seconds?: number; + convergence_p75_intvty_requests?: number; + convergence_p75_intvty_min?: number; + convergence_p75_intvty_max?: number; + convergence_p75_intvty_max_relative_deviation?: number; + convergence_p90_intvty_time_seconds?: number; + convergence_p90_intvty_requests?: number; + convergence_p90_intvty_min?: number; + convergence_p90_intvty_max?: number; + convergence_p90_intvty_max_relative_deviation?: number; } /** @@ -272,9 +310,14 @@ export interface InferenceData extends Partial ({ hwKey, precision, x, y, tp, conc: 16, framework: 'vllm' }) as unknown as InferenceData; -const rangedPoint = ( +const convergedPoint = ( hwKey: string, x: number, y: number, @@ -63,10 +63,12 @@ const rangedPoint = ( max: number, ): InferenceData => ({ ...point(hwKey, 'fp8', x, y, 8), - observedXMin: min, - observedXMax: max, - observed_window_seconds: 600, - observed_window_count: 6, + convergenceXMin: min, + convergenceXMax: max, + convergenceTimeSeconds: 1200, + convergenceRequests: 407, + convergenceMaxRelativeDeviation: 0.04, + convergence_tolerance_ratio: 0.05, }); // h100 owns both axis extremes so hiding b200 / showing fp4 keeps the niced @@ -235,12 +237,12 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); - it('renders observed ranges only for Pareto-front points and fits their x extent', () => { + it('renders convergence ranges only for Pareto-front points and fits their x extent', () => { const data = [ - rangedPoint('h100', 1, 50, 0.5, 1.8), - rangedPoint('h100', 2, 100, 1.2, 8), + convergedPoint('h100', 1, 50, 0.5, 1.8), + convergedPoint('h100', 2, 100, 1.2, 8), // Below the running upper-right envelope, so it must not get a whisker. - rangedPoint('h100', 1.5, 40, 0.8, 6), + convergedPoint('h100', 1.5, 40, 0.8, 6), ]; const { container, unmount } = mountChart({ data, @@ -251,7 +253,7 @@ describe('ScatterGraph toggle decoration', () => { }); const ranges = [ - ...container.querySelectorAll('.observed-window-range[data-source="official"]'), + ...container.querySelectorAll('.convergence-range[data-source="official"]'), ]; expect(ranges).toHaveLength(2); expect(ranges.every((range) => range.style.pointerEvents === 'none')).toBe(true); @@ -266,7 +268,7 @@ describe('ScatterGraph toggle decoration', () => { (range) => (range as SVGGElement & { __data__: { point: InferenceData } }).__data__.point.x === 2, )!; - const stem = wideRange.querySelector('.observed-window-range-stem')!; + const stem = wideRange.querySelector('.convergence-range-stem')!; const pointDot = dotGroups(container).find( (dot) => (dot as SVGGElement & { __data__: InferenceData }).__data__.x === 2, )!; @@ -274,12 +276,25 @@ describe('ScatterGraph toggle decoration', () => { pointDot.getAttribute('transform')?.match(/translate\((?[^,]+)/u)?.groups?.x, ); expect(Number(stem.getAttribute('x2'))).toBeGreaterThan(pointX); - // The observed max participates in the domain, so its cap remains inside + // The convergence max participates in the domain, so its cap remains inside // the 800px chart rather than being clipped at the right edge. expect(Number(stem.getAttribute('x2'))).toBeLessThan(800); unmount(); }); + it('renders no whisker for a point that was evaluated but did not stabilize', () => { + const unstable = { + ...point('h100', 'fp8', 2, 100, 8), + convergenceEvaluated: true, + convergence_tolerance_ratio: 0.05, + convergence_horizon_seconds: 3600, + }; + const { container, unmount } = mountChart({ data: [unstable] }); + + expect(container.querySelectorAll('.convergence-range')).toHaveLength(0); + unmount(); + }); + it('hides a toggled-off hw via opacity without rebuilding the chart', () => { const { container, rerender, unmount } = mountChart(); const buildsAfterMount = rebuildCount(); @@ -442,10 +457,10 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); - it('renders and removes overlay observed ranges through the overlay path', () => { + it('renders and removes overlay convergence ranges through the overlay path', () => { const overlayPoints = [ - rangedPoint('h100', 30, 250, 20, 36), - rangedPoint('h100', 35, 300, 28, 48), + convergedPoint('h100', 30, 250, 20, 36), + convergedPoint('h100', 35, 300, 28, 48), ].map((p) => ({ ...p, run_url: 'https://github.com/o/r/actions/runs/123', @@ -473,15 +488,13 @@ describe('ScatterGraph toggle decoration', () => { }); const ranges = [ - ...container.querySelectorAll('.observed-window-range[data-source="overlay"]'), + ...container.querySelectorAll('.convergence-range[data-source="overlay"]'), ]; expect(ranges).toHaveLength(2); expect(ranges.every((range) => range.getAttribute('stroke') === overlayRunColor(0))).toBe(true); updateProps({ overlayData: undefined }); - expect( - container.querySelectorAll('.observed-window-range[data-source="overlay"]'), - ).toHaveLength(0); + expect(container.querySelectorAll('.convergence-range[data-source="overlay"]')).toHaveLength(0); unmount(); }); diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 1124f81c..51eff7d3 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -262,9 +262,9 @@ interface ScaleConfigValue { _isLog?: boolean; } -interface ObservedRangeMark { +interface ConvergenceRangeMark { key: string; - point: InferenceData & { observedXMin: number; observedXMax: number }; + point: InferenceData & { convergenceXMin: number; convergenceXMax: number }; hw: string; precision: string; source: 'official' | 'overlay'; @@ -273,33 +273,28 @@ interface ObservedRangeMark { runIndex?: number; } -const hasObservedWindowRange = ( +const hasConvergenceRange = ( point: InferenceData, -): point is InferenceData & { observedXMin: number; observedXMax: number } => - (point.observed_window_count ?? 0) >= 2 && - typeof point.observedXMin === 'number' && - Number.isFinite(point.observedXMin) && - point.observedXMin > 0 && - typeof point.observedXMax === 'number' && - Number.isFinite(point.observedXMax) && - point.observedXMax > point.observedXMin; - -const observedRangeLegendLabel = ( +): point is InferenceData & { convergenceXMin: number; convergenceXMax: number } => + typeof point.convergenceTimeSeconds === 'number' && + Number.isFinite(point.convergenceTimeSeconds) && + point.convergenceTimeSeconds > 0 && + typeof point.convergenceXMin === 'number' && + Number.isFinite(point.convergenceXMin) && + point.convergenceXMin > 0 && + typeof point.convergenceXMax === 'number' && + Number.isFinite(point.convergenceXMax) && + point.convergenceXMax > point.convergenceXMin; + +const convergenceRangeLegendLabel = ( locale: 'en' | 'zh', - windowSeconds: number | undefined, + toleranceRatio: number | undefined, ): string => { - if (!windowSeconds) { - return locale === 'zh' - ? '观测时间窗口范围——并非置信区间' - : 'Observed window range — not a confidence interval'; - } - const minutes = windowSeconds / 60; - if (locale === 'zh') { - const duration = Number.isInteger(minutes) ? `${minutes} 分钟` : `${windowSeconds} 秒`; - return `${duration}观测范围——并非置信区间`; - } - const duration = Number.isInteger(minutes) ? `${minutes}-minute` : `${windowSeconds}-second`; - return `Observed ${duration} range — not a confidence interval`; + const tolerance = + typeof toleranceRatio === 'number' ? `±${Number((toleranceRatio * 100).toFixed(2))}%` : ''; + return locale === 'zh' + ? `累计指标在${tolerance ? ` ${tolerance} ` : ''}内稳定后的范围——回溯性诊断，并非置信区间` + : `Cumulative span after ${tolerance ? `${tolerance} ` : ''}stabilization — retrospective, not a confidence interval`; }; const isSameScaleConfig = (a: ScaleConfigValue, b: ScaleConfigValue): boolean => a.type === b.type && @@ -1001,24 +996,21 @@ const ScatterGraph = React.memo( activeOverlayHwTypes, ]); - const observedRangePoints = useMemo( - () => visibleFrontierPoints.filter(hasObservedWindowRange), + const convergenceRangePoints = useMemo( + () => visibleFrontierPoints.filter(hasConvergenceRange), [visibleFrontierPoints], ); - // A mixed-duration comparison gets a generic key rather than claiming one - // window width applies to every point. Current AgentX artifacts use 600s. - const observedRangeWindowSeconds = useMemo(() => { - const durations = new Set( - observedRangePoints - .map((point) => point.observed_window_seconds) - .filter( - (seconds): seconds is number => - typeof seconds === 'number' && Number.isFinite(seconds) && seconds > 0, - ), + // A mixed-policy comparison gets a generic key rather than claiming one + // tolerance applies to every point. + const convergenceToleranceRatio = useMemo(() => { + const tolerances = new Set( + convergenceRangePoints + .map((point) => point.convergence_tolerance_ratio) + .filter((ratio): ratio is number => typeof ratio === 'number' && Number.isFinite(ratio)), ); - return durations.size === 1 ? [...durations][0] : undefined; - }, [observedRangePoints]); + return tolerances.size === 1 ? [...tolerances][0] : undefined; + }, [convergenceRangePoints]); // Ref for trackedConfigIds (needs to be current at event time inside D3 handlers) const trackedConfigIdsRef = useRef(trackedConfigIds); @@ -1053,7 +1045,10 @@ const ScatterGraph = React.memo( (visiblePoints.length > 0 ? (d3.extent([ ...visiblePoints.map((point) => point.x), - ...observedRangePoints.flatMap((point) => [point.observedXMin, point.observedXMax]), + ...convergenceRangePoints.flatMap((point) => [ + point.convergenceXMin, + point.convergenceXMax, + ]), ]) as [number, number]) : ([0, 100] as [number, number])); @@ -1076,7 +1071,7 @@ const ScatterGraph = React.memo( }; }, [ visiblePoints, - observedRangePoints, + convergenceRangePoints, isInputTputMetric, xLabel, scaleType, @@ -1155,7 +1150,7 @@ const ScatterGraph = React.memo( [effectiveActiveHwTypes, selectedPrecisions], ); - const observedRangeOpacity = useCallback( + const convergenceRangeOpacity = useCallback( (el: SVGGElement): number => { const hw = el.dataset.hwKey; const precision = el.dataset.precision; @@ -1226,9 +1221,9 @@ const ScatterGraph = React.memo( return this.dataset.hwKey === hwKey ? null : '0.15'; }); root - .selectAll('.observed-window-range[data-source="official"]') + .selectAll('.convergence-range[data-source="official"]') .style('opacity', function () { - const base = observedRangeOpacity(this); + const base = convergenceRangeOpacity(this); if (base === 0) return 0; return this.dataset.hwKey === hwKey ? base : 0.06; }); @@ -1238,7 +1233,7 @@ const ScatterGraph = React.memo( return labelOpacityForHover((this as SVGGElement).dataset, hwKey); }); }, - [isPointVisible, isRooflineVisible, observedRangeOpacity], + [isPointVisible, isRooflineVisible, convergenceRangeOpacity], ); const handleLegendHoverEnd = useCallback(() => { @@ -1251,8 +1246,8 @@ const ScatterGraph = React.memo( root.selectAll('.roofline-path').style('opacity', function () { return isRooflineVisible(this) ? 1 : 0; }); - root.selectAll('.observed-window-range').style('opacity', function () { - return observedRangeOpacity(this); + root.selectAll('.convergence-range').style('opacity', function () { + return convergenceRangeOpacity(this); }); root .selectAll('.parallelism-label, .line-label') @@ -1266,7 +1261,7 @@ const ScatterGraph = React.memo( }, [ isPointVisible, isRooflineVisible, - observedRangeOpacity, + convergenceRangeOpacity, effectiveActiveHwTypes, selectedPrecisions, ]); @@ -1529,19 +1524,19 @@ const ScatterGraph = React.memo( .style('transition', 'opacity 150ms ease') .style('opacity', (d) => (d.visible ? 1 : 0)); - // Thin, capped horizontal whiskers summarize the range observed - // across non-overlapping windows in this one run. They deliberately - // live in the roofline layer (behind points), carry no pointer - // events, and are never used to compute the Pareto frontier. - const observedRangeMarks: ObservedRangeMark[] = []; + // Thin, capped horizontal whiskers summarize cumulative estimates + // after retrospective stabilization. They deliberately live in the + // roofline layer (behind points), carry no pointer events, and are + // never used to compute the Pareto frontier. + const convergenceRangeMarks: ConvergenceRangeMark[] = []; for (const [key, front] of Object.entries(rooflines)) { const hw = key.split('_').slice(0, -1).join('_'); const precision = key.split('_').pop()!; const visible = ir.effectiveActiveHwTypes.has(hw) && ir.selectedPrecisions.includes(precision); front.forEach((point, index) => { - if (!hasObservedWindowRange(point)) return; - observedRangeMarks.push({ + if (!hasConvergenceRange(point)) return; + convergenceRangeMarks.push({ key: `official-${key}-${point.date}-${point.id ?? 'na'}-${point.conc}-${point.x}-${point.y}-${index}`, point, hw, @@ -1556,8 +1551,8 @@ const ScatterGraph = React.memo( for (const [key, group] of Object.entries(overlayRooflines)) { if (!overlayData.hardwareConfig[group.hwKey]) continue; group.points.forEach((point, index) => { - if (!hasObservedWindowRange(point)) return; - observedRangeMarks.push({ + if (!hasConvergenceRange(point)) return; + convergenceRangeMarks.push({ key: `overlay-${key}-${point.date}-${point.id ?? 'na'}-${point.conc}-${point.x}-${point.y}-${index}`, point, hw: group.hwKey, @@ -1573,21 +1568,21 @@ const ScatterGraph = React.memo( } } - const observedRanges = rooflinesLayer - .selectAll('.observed-window-range') - .data(observedRangeMarks, (mark) => mark.key) + const convergenceRanges = rooflinesLayer + .selectAll('.convergence-range') + .data(convergenceRangeMarks, (mark) => mark.key) .join( (enter) => { - const group = enter.append('g').attr('class', 'observed-window-range'); - group.append('line').attr('class', 'observed-window-range-stem'); - group.append('line').attr('class', 'observed-window-range-cap-min'); - group.append('line').attr('class', 'observed-window-range-cap-max'); + const group = enter.append('g').attr('class', 'convergence-range'); + group.append('line').attr('class', 'convergence-range-stem'); + group.append('line').attr('class', 'convergence-range-cap-min'); + group.append('line').attr('class', 'convergence-range-cap-max'); return group; }, (update) => update, (exit) => exit.remove(), ) - .attr('data-testid', 'observed-window-range') + .attr('data-testid', 'convergence-range') .attr('data-source', (mark) => mark.source) .attr('data-hw-key', (mark) => mark.hw) .attr('data-precision', (mark) => mark.precision) @@ -1601,32 +1596,32 @@ const ScatterGraph = React.memo( .style('transition', 'opacity 150ms ease') .style('opacity', (mark) => (mark.visible ? 0.42 : 0)); - observedRanges.selectAll('line').attr('vector-effect', 'non-scaling-stroke'); - observedRanges.each(function (mark) { + convergenceRanges.selectAll('line').attr('vector-effect', 'non-scaling-stroke'); + convergenceRanges.each(function (mark) { const group = d3.select(this); - const minX = xScale(mark.point.observedXMin); - const maxX = xScale(mark.point.observedXMax); + const minX = xScale(mark.point.convergenceXMin); + const maxX = xScale(mark.point.convergenceXMax); const y = yScale(mark.point.y); group - .select('.observed-window-range-stem') + .select('.convergence-range-stem') .attr('x1', minX) .attr('x2', maxX) .attr('y1', y) .attr('y2', y); group - .select('.observed-window-range-cap-min') + .select('.convergence-range-cap-min') .attr('x1', minX) .attr('x2', minX) .attr('y1', y - 3.5) .attr('y2', y + 3.5); group - .select('.observed-window-range-cap-max') + .select('.convergence-range-cap-max') .attr('x1', maxX) .attr('x2', maxX) .attr('y1', y - 3.5) .attr('y2', y + 3.5); }); - observedRanges.lower(); + convergenceRanges.lower(); // Parallelism labels interface LabelSeg { @@ -2079,29 +2074,29 @@ const ScatterGraph = React.memo( } }); - // Keep observed-window whiskers anchored to the full-run point while + // Keep convergence whiskers anchored to the full-run point while // zooming. End caps stay a fixed seven CSS pixels tall. zoomGroup - .selectAll('.observed-window-range') + .selectAll('.convergence-range') .each(function (mark) { const group = d3.select(this); - const minX = newXScale(mark.point.observedXMin); - const maxX = newXScale(mark.point.observedXMax); + const minX = newXScale(mark.point.convergenceXMin); + const maxX = newXScale(mark.point.convergenceXMax); const y = newYScale(mark.point.y); group - .select('.observed-window-range-stem') + .select('.convergence-range-stem') .attr('x1', minX) .attr('x2', maxX) .attr('y1', y) .attr('y2', y); group - .select('.observed-window-range-cap-min') + .select('.convergence-range-cap-min') .attr('x1', minX) .attr('x2', minX) .attr('y1', y - 3.5) .attr('y2', y + 3.5); group - .select('.observed-window-range-cap-max') + .select('.convergence-range-cap-max') .attr('x1', maxX) .attr('x2', maxX) .attr('y1', y - 3.5) @@ -2937,11 +2932,11 @@ const ScatterGraph = React.memo( } }); - // Observed-window ranges follow the same visibility/recolor path as + // Convergence ranges follow the same visibility/recolor path as // their frontier series without touching their zoomed geometry. - zoomGroup.selectAll('.observed-window-range').each(function () { + zoomGroup.selectAll('.convergence-range').each(function () { const el = d3.select(this); - el.style('opacity', observedRangeOpacity(this)); + el.style('opacity', convergenceRangeOpacity(this)); if (this.dataset.source === 'official' && this.dataset.hwKey) { el.attr('stroke', ir.getCssColor(ir.resolveColor(this.dataset.hwKey))); } @@ -3004,7 +2999,7 @@ const ScatterGraph = React.memo( showGradientLabels, showLineLabels, gradientColorByPoint, - observedRangeOpacity, + convergenceRangeOpacity, ]); // D3 custom layers are keyed additions, so removing the overlay layer from @@ -3016,7 +3011,7 @@ const ScatterGraph = React.memo( if (!svg) return; d3.select(svg) .selectAll( - '.unofficial-overlay-pt, .overlay-roofline-path, .observed-window-range[data-source="overlay"]', + '.unofficial-overlay-pt, .overlay-roofline-path, .convergence-range[data-source="overlay"]', ) .remove(); }, [overlayData]); @@ -3207,9 +3202,9 @@ const ScatterGraph = React.memo( ]} disableActiveSort={false} keyIndicators={ - observedRangePoints.length > 0 ? ( + convergenceRangePoints.length > 0 ? (
- {observedRangeLegendLabel(locale, observedRangeWindowSeconds)} + {convergenceRangeLegendLabel(locale, convergenceToleranceRatio)}
) : undefined } diff --git a/packages/app/src/components/inference/utils.test.ts b/packages/app/src/components/inference/utils.test.ts index 7ff03720..394b4489 100644 --- a/packages/app/src/components/inference/utils.test.ts +++ b/packages/app/src/components/inference/utils.test.ts @@ -233,22 +233,28 @@ describe('processOverlayChartData', () => { expect(result[0].x).toBe(0.12); }); - it('stamps the observed bounds matching the overlay x-axis metric', () => { + it('stamps convergence bounds matching the overlay x-axis metric', () => { const data = [ pt({ tpPerGpu: { y: 42, roof: false }, p90_ttft: 2, - observed_window_count: 6, - observed_window_p90_ttft_min: 1.4, - observed_window_p90_ttft_max: 4.1, - observed_window_p75_e2el_min: 40, - observed_window_p75_e2el_max: 80, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: 407, + convergence_p90_ttft_min: 1.4, + convergence_p90_ttft_max: 1.52, + convergence_p90_ttft_max_relative_deviation: 0.043, } as any), ]; const result = processOverlayChartData(data, 'e2e', 'y_tpPerGpu', 'p90_ttft'); - expect(result[0].observedXMin).toBe(1.4); - expect(result[0].observedXMax).toBe(4.1); + expect(result[0].convergenceXMin).toBe(1.4); + expect(result[0].convergenceXMax).toBe(1.52); + expect(result[0].convergenceTimeSeconds).toBe(1200); + expect(result[0].convergenceRequests).toBe(407); }); it('filters e2e TTFT outliers exceeding y_latency_limit', () => { diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index c54d26b6..8df884cc 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -7,7 +7,7 @@ import chartDefinitions from '@/components/inference/inference-chart-config.json'; import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; -import { withObservedWindowRange } from '@/components/inference/utils/observed-window-range'; +import { withConvergenceRange } from '@/components/inference/utils/convergence-range'; import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; @@ -128,7 +128,7 @@ export function processOverlayChartData( .map((d: InferenceData) => { const yValue = (d[metricKey] as { y: number })?.y ?? d.y; const xValue = (d as any)[xAxisField] ?? d.x; - return withObservedWindowRange({ ...d, x: xValue, y: yValue }, xAxisField); + return withConvergenceRange({ ...d, x: xValue, y: yValue }, xAxisField); }) .filter( (d) => !isTtftX || isAgentic || !chartDef.y_latency_limit || d.x <= chartDef.y_latency_limit, diff --git a/packages/app/src/components/inference/utils/convergence-range.test.ts b/packages/app/src/components/inference/utils/convergence-range.test.ts new file mode 100644 index 00000000..1aa9c9ab --- /dev/null +++ b/packages/app/src/components/inference/utils/convergence-range.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; + +import type { InferenceData } from '@/components/inference/types'; + +import { + convergenceEvaluatedForXAxis, + convergenceRangeForXAxis, + withConvergenceRange, +} from './convergence-range'; + +const diagnostic = { + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, +}; + +describe('convergenceRangeForXAxis', () => { + it('selects the stabilization result matching the resolved x-axis field', () => { + const point = { + ...diagnostic, + convergence_p90_ttft_time_seconds: 900, + convergence_p90_ttft_requests: 312, + convergence_p90_ttft_min: 1.4, + convergence_p90_ttft_max: 1.52, + convergence_p90_ttft_max_relative_deviation: 0.041, + convergence_p75_e2el_time_seconds: 1200, + convergence_p75_e2el_requests: 407, + convergence_p75_e2el_min: 40, + convergence_p75_e2el_max: 42, + convergence_p75_e2el_max_relative_deviation: 0.03, + }; + + expect(convergenceRangeForXAxis(point, 'p90_ttft')).toEqual({ + min: 1.4, + max: 1.52, + timeSeconds: 900, + requests: 312, + maxRelativeDeviation: 0.041, + }); + expect(convergenceRangeForXAxis(point, 'p75_e2el')?.timeSeconds).toBe(1200); + }); + + it('distinguishes an evaluated metric that did not stabilize from missing diagnostics', () => { + expect(convergenceEvaluatedForXAxis(diagnostic, 'p90_intvty')).toBe(true); + expect(convergenceRangeForXAxis(diagnostic, 'p90_intvty')).toBeNull(); + expect(convergenceEvaluatedForXAxis({}, 'p90_intvty')).toBe(false); + expect(convergenceEvaluatedForXAxis(diagnostic, 'median_intvty')).toBe(false); + }); + + it('rejects invalid bounds and metadata while allowing a flat stabilized span', () => { + const base = { + ...diagnostic, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: 407, + convergence_p90_ttft_min: 2, + convergence_p90_ttft_max: 2, + convergence_p90_ttft_max_relative_deviation: 0, + }; + expect(convergenceRangeForXAxis(base, 'p90_ttft')).toMatchObject({ min: 2, max: 2 }); + expect( + convergenceRangeForXAxis({ ...base, convergence_p90_ttft_min: 0 }, 'p90_ttft'), + ).toBeNull(); + expect( + convergenceRangeForXAxis({ ...base, convergence_p90_ttft_max: 1 }, 'p90_ttft'), + ).toBeNull(); + expect( + convergenceRangeForXAxis( + { ...base, convergence_p90_ttft_max_relative_deviation: Number.NaN }, + 'p90_ttft', + ), + ).toBeNull(); + }); +}); + +describe('withConvergenceRange', () => { + it('stamps chart-ready convergence metadata and clears it when the x field changes', () => { + const point = { + x: 2, + y: 100, + date: '2026-07-21', + tp: 8, + conc: 16, + precision: 'fp4', + hwKey: 'b300-vllm', + tpPerGpu: { y: 100, roof: true }, + tpPerMw: { y: 1, roof: true }, + costh: { y: 1, roof: true }, + costn: { y: 1, roof: true }, + costr: { y: 1, roof: true }, + costhi: { y: 1, roof: true }, + costni: { y: 1, roof: true }, + costri: { y: 1, roof: true }, + ...diagnostic, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: 407, + convergence_p90_ttft_min: 1, + convergence_p90_ttft_max: 3, + convergence_p90_ttft_max_relative_deviation: 0.04, + } as InferenceData; + + const stamped = withConvergenceRange(point, 'p90_ttft'); + expect(stamped.convergenceEvaluated).toBe(true); + expect(stamped.convergenceXMin).toBe(1); + expect(stamped.convergenceXMax).toBe(3); + expect(stamped.convergenceTimeSeconds).toBe(1200); + expect(stamped.convergenceRequests).toBe(407); + + const cleared = withConvergenceRange(stamped, 'median_ttft'); + expect(cleared.convergenceEvaluated).toBe(false); + expect(cleared.convergenceXMin).toBeUndefined(); + expect(cleared.convergenceTimeSeconds).toBeUndefined(); + }); +}); diff --git a/packages/app/src/components/inference/utils/convergence-range.ts b/packages/app/src/components/inference/utils/convergence-range.ts new file mode 100644 index 00000000..b642a8b4 --- /dev/null +++ b/packages/app/src/components/inference/utils/convergence-range.ts @@ -0,0 +1,78 @@ +import type { AggDataEntry, InferenceData } from '@/components/inference/types'; + +export interface ConvergenceRange { + min: number; + max: number; + timeSeconds: number; + requests: number; + maxRelativeDeviation: number; +} + +const CONVERGENCE_X_FIELD = /^(?:p75|p90)_(?:ttft|e2el|intvty)$/u; + +const finiteNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); + +/** Whether this run evaluated cumulative convergence for the selected x field. */ +export function convergenceEvaluatedForXAxis( + point: Partial, + xAxisField: string, +): boolean { + return ( + CONVERGENCE_X_FIELD.test(xAxisField) && + finiteNumber(point.convergence_checkpoint_seconds) && + point.convergence_checkpoint_seconds > 0 && + finiteNumber(point.convergence_tolerance_ratio) && + point.convergence_tolerance_ratio >= 0 && + finiteNumber(point.convergence_min_confirmation_seconds) && + point.convergence_min_confirmation_seconds >= 0 && + finiteNumber(point.convergence_horizon_seconds) && + point.convergence_horizon_seconds > 0 + ); +} + +/** Resolve post-stabilization cumulative bounds for the chart's current x field. */ +export function convergenceRangeForXAxis( + point: Partial, + xAxisField: string, +): ConvergenceRange | null { + if (!convergenceEvaluatedForXAxis(point, xAxisField)) return null; + + const values = point as unknown as Record; + const prefix = `convergence_${xAxisField}`; + const min = values[`${prefix}_min`]; + const max = values[`${prefix}_max`]; + const timeSeconds = values[`${prefix}_time_seconds`]; + const requests = values[`${prefix}_requests`]; + const maxRelativeDeviation = values[`${prefix}_max_relative_deviation`]; + if ( + !finiteNumber(min) || + min <= 0 || + !finiteNumber(max) || + max < min || + !finiteNumber(timeSeconds) || + timeSeconds <= 0 || + !finiteNumber(requests) || + requests < 0 || + !finiteNumber(maxRelativeDeviation) || + maxRelativeDeviation < 0 + ) { + return null; + } + + return { min, max, timeSeconds, requests, maxRelativeDeviation }; +} + +/** Stamp the selected convergence result onto a chart-ready point. */ +export function withConvergenceRange(point: InferenceData, xAxisField: string): InferenceData { + const range = convergenceRangeForXAxis(point, xAxisField); + return { + ...point, + convergenceEvaluated: convergenceEvaluatedForXAxis(point, xAxisField), + convergenceXMin: range?.min, + convergenceXMax: range?.max, + convergenceTimeSeconds: range?.timeSeconds, + convergenceRequests: range?.requests, + convergenceMaxRelativeDeviation: range?.maxRelativeDeviation, + }; +} diff --git a/packages/app/src/components/inference/utils/observed-window-range.test.ts b/packages/app/src/components/inference/utils/observed-window-range.test.ts deleted file mode 100644 index 50205535..00000000 --- a/packages/app/src/components/inference/utils/observed-window-range.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import type { InferenceData } from '@/components/inference/types'; - -import { observedWindowRangeForXAxis, withObservedWindowRange } from './observed-window-range'; - -describe('observedWindowRangeForXAxis', () => { - it('selects bounds matching the resolved x-axis field', () => { - const point = { - observed_window_count: 6, - observed_window_p90_ttft_min: 1.4, - observed_window_p90_ttft_max: 4.1, - observed_window_p75_e2el_min: 40, - observed_window_p75_e2el_max: 80, - }; - - expect(observedWindowRangeForXAxis(point, 'p90_ttft')).toEqual({ min: 1.4, max: 4.1 }); - expect(observedWindowRangeForXAxis(point, 'p75_e2el')).toEqual({ min: 40, max: 80 }); - }); - - it('requires at least two observed windows', () => { - expect( - observedWindowRangeForXAxis( - { - observed_window_count: 1, - observed_window_p90_ttft_min: 1, - observed_window_p90_ttft_max: 2, - }, - 'p90_ttft', - ), - ).toBeNull(); - }); - - it('rejects missing, non-positive, reversed, and degenerate bounds', () => { - expect(observedWindowRangeForXAxis({ observed_window_count: 6 }, 'p90_ttft')).toBeNull(); - expect( - observedWindowRangeForXAxis( - { - observed_window_count: 6, - observed_window_p90_ttft_min: 0, - observed_window_p90_ttft_max: 2, - }, - 'p90_ttft', - ), - ).toBeNull(); - expect( - observedWindowRangeForXAxis( - { - observed_window_count: 6, - observed_window_p90_ttft_min: 2, - observed_window_p90_ttft_max: 2, - }, - 'p90_ttft', - ), - ).toBeNull(); - }); -}); - -describe('withObservedWindowRange', () => { - it('stamps chart-ready bounds and clears stale bounds when the x field changes', () => { - const point = { - x: 2, - y: 100, - date: '2026-07-21', - tp: 8, - conc: 16, - precision: 'fp4', - hwKey: 'b300-vllm', - tpPerGpu: { y: 100, roof: true }, - tpPerMw: { y: 1, roof: true }, - costh: { y: 1, roof: true }, - costn: { y: 1, roof: true }, - costr: { y: 1, roof: true }, - costhi: { y: 1, roof: true }, - costni: { y: 1, roof: true }, - costri: { y: 1, roof: true }, - observed_window_count: 6, - observed_window_p90_ttft_min: 1, - observed_window_p90_ttft_max: 3, - } as InferenceData; - - const stamped = withObservedWindowRange(point, 'p90_ttft'); - expect(stamped.observedXMin).toBe(1); - expect(stamped.observedXMax).toBe(3); - - const cleared = withObservedWindowRange(stamped, 'median_ttft'); - expect(cleared.observedXMin).toBeUndefined(); - expect(cleared.observedXMax).toBeUndefined(); - }); -}); diff --git a/packages/app/src/components/inference/utils/observed-window-range.ts b/packages/app/src/components/inference/utils/observed-window-range.ts deleted file mode 100644 index 93d078a0..00000000 --- a/packages/app/src/components/inference/utils/observed-window-range.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AggDataEntry, InferenceData } from '@/components/inference/types'; - -export interface ObservedWindowRange { - min: number; - max: number; -} - -/** - * Resolve the descriptive window bounds matching the chart's current x field. - * Missing, degenerate, and single-window diagnostics intentionally render no - * whisker: there is no within-run range to communicate in those cases. - */ -export function observedWindowRangeForXAxis( - point: Partial, - xAxisField: string, -): ObservedWindowRange | null { - if ((point.observed_window_count ?? 0) < 2) return null; - const values = point as unknown as Record; - const min = values[`observed_window_${xAxisField}_min`]; - const max = values[`observed_window_${xAxisField}_max`]; - if ( - typeof min !== 'number' || - typeof max !== 'number' || - !Number.isFinite(min) || - !Number.isFinite(max) || - min <= 0 || - max <= min - ) { - return null; - } - return { min, max }; -} - -export function withObservedWindowRange(point: InferenceData, xAxisField: string): InferenceData { - const range = observedWindowRangeForXAxis(point, xAxisField); - return { - ...point, - observedXMin: range?.min, - observedXMax: range?.max, - }; -} diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index 301bb488..b7e319be 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -71,15 +71,19 @@ function tooltipConfig(overrides: Partial = {}): TooltipConfig { }; } -function observedRangePoint(overrides: Partial = {}): InferenceData { +function convergencePoint(overrides: Partial = {}): InferenceData { return pt({ benchmark_type: 'agentic_traces', - observedXMin: 1.416, - observedXMax: 4.114, - observed_window_seconds: 600, - observed_window_expected_count: 6, - observed_window_count: 6, - observed_window_min_requests: 21, + convergenceEvaluated: true, + convergenceXMin: 43.36036, + convergenceXMax: 46.97824, + convergenceTimeSeconds: 1200, + convergenceRequests: 407, + convergenceMaxRelativeDeviation: 0.04385, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, root_trajectory_count: 3, root_trajectory_kish_effective_count: 1.69, ...overrides, @@ -441,37 +445,58 @@ describe('generateTooltipContent', () => { expect(html).not.toContain('Untrack Over Time'); }); - it('labels the observed window range as descriptive, not inferential', () => { - const html = generateTooltipContent(tooltipConfig({ data: observedRangePoint() })); + it('explains cumulative stabilization without making inferential claims', () => { + const html = generateTooltipContent(tooltipConfig({ data: convergencePoint() })); - expect(html).toContain('Observed 10-minute range: 1.416–4.114'); expect(html).toContain( - '6 non-overlapping windows; not a confidence interval or rerun prediction.', + 'Cumulative convergence: Stabilized by 20 minutes at ±5%', ); + expect(html).toContain('Post-stabilization span: 43.36–46.978'); + expect(html).toContain('Requests at stabilization: 407'); + expect(html).toContain('Maximum later deviation: 4.38%'); + expect(html).toContain('5-minute cumulative checkpoints'); + expect(html).toContain('at least 20 minutes of later confirmation'); + expect(html).toContain('not a confidence interval or rerun prediction'); expect(html).toContain('Root trajectories: 3'); expect(html).toContain('Kish-effective root coverage: 1.69'); expect(html).toContain('not a statistical effective sample size'); - expect(html).toContain('Smallest window: 21 successful requests'); }); - it('renders natural Chinese observed-range caveats', () => { - const html = generateTooltipContent( - tooltipConfig({ locale: 'zh', data: observedRangePoint() }), - ); + it('renders natural Chinese convergence language', () => { + const html = generateTooltipContent(tooltipConfig({ locale: 'zh', data: convergencePoint() })); - expect(html).toContain('10 分钟观测范围: 1.416–4.114'); - expect(html).toContain('基于 6 个互不重叠的时间窗口；并非置信区间，也不预测复跑结果。'); + expect(html).toContain('累计收敛: 在 20 分钟时已稳定至 ±5% 范围内'); + expect(html).toContain('稳定后的累计范围: 43.36–46.978'); + expect(html).toContain('达到稳定时的请求数: 407'); + expect(html).toContain('后续最大偏差: 4.38%'); + expect(html).toContain('回溯性单次运行诊断'); + expect(html).toContain('并非置信区间，也不预测复跑结果'); expect(html).toContain('根轨迹数: 3'); expect(html).toContain('Kish 有效根轨迹覆盖数: 1.69'); expect(html).toContain('并非统计有效样本量'); - expect(html).toContain('最小时间窗口: 21个成功请求'); }); - it('omits an observed range when fewer than two windows are available', () => { + it('marks an evaluated metric that never stabilized and shows no span', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: convergencePoint({ + convergenceTimeSeconds: undefined, + convergenceXMin: undefined, + convergenceXMax: undefined, + convergenceRequests: undefined, + convergenceMaxRelativeDeviation: undefined, + }), + }), + ); + expect(html).toContain('Not stabilized within the 60-minute run at ±5%'); + expect(html).not.toContain('Post-stabilization span'); + }); + + it('omits convergence copy when the selected metric was not evaluated', () => { const html = generateTooltipContent( - tooltipConfig({ data: observedRangePoint({ observed_window_count: 1 }) }), + tooltipConfig({ data: convergencePoint({ convergenceEvaluated: false }) }), ); - expect(html).not.toContain('observed-window-range-tooltip'); + expect(html).not.toContain('convergence-diagnostic-tooltip'); }); }); @@ -555,10 +580,11 @@ describe('generateOverlayTooltipContent', () => { expect(html).not.toContain('CPU Cache Hit Rate'); }); - it('shows the same observed-range caveat for unofficial overlays', () => { - const html = generateOverlayTooltipContent(overlayConfig({ data: observedRangePoint() })); + it('shows the same convergence caveat for unofficial overlays', () => { + const html = generateOverlayTooltipContent(overlayConfig({ data: convergencePoint() })); - expect(html).toContain('Observed 10-minute range: 1.416–4.114'); + expect(html).toContain('Stabilized by 20 minutes at ±5%'); + expect(html).toContain('Post-stabilization span'); expect(html).toContain('not a confidence interval or rerun prediction'); expect(html).toContain('Kish-effective root coverage'); }); diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 7a5d97fe..3758cfc6 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -99,75 +99,96 @@ export const fmt = (v: number): string => { return String(rounded); }; -const OBSERVED_RANGE_STRINGS = { +const CONVERGENCE_STRINGS = { en: { - genericTitle: 'Observed window range', - title: (duration: string) => `Observed ${duration} range`, - completeWindows: (count: number) => - `${count} non-overlapping windows; not a confidence interval or rerun prediction.`, - partialWindows: (count: number, expected: number) => - `${count} of ${expected} non-overlapping windows contained successful requests; not a confidence interval or rerun prediction.`, + title: 'Cumulative convergence', + stabilized: (duration: string, tolerance: string) => + `Stabilized by ${duration} at ${tolerance}`, + notStabilized: (duration: string, tolerance: string) => + `Not stabilized within the ${duration} run at ${tolerance}`, + span: 'Post-stabilization span', + requests: 'Requests at stabilization', + maxDeviation: 'Maximum later deviation', + method: (checkpoint: string, confirmation: string) => + `Retrospective within-run diagnostic using ${checkpoint} cumulative checkpoints and at least ${confirmation} of later confirmation; not a confidence interval or rerun prediction.`, rootTrajectories: 'Root trajectories', kishCoverage: 'Kish-effective root coverage', kishExplanation: 'coverage diversity, not a statistical effective sample size', - smallestWindow: 'Smallest window', - successfulRequests: 'successful requests', }, zh: { - genericTitle: '观测时间窗口范围', - title: (duration: string) => `${duration}观测范围`, - completeWindows: (count: number) => - `基于 ${count} 个互不重叠的时间窗口；并非置信区间，也不预测复跑结果。`, - partialWindows: (count: number, expected: number) => - `${expected} 个互不重叠的时间窗口中有 ${count} 个包含成功请求；并非置信区间，也不预测复跑结果。`, + title: '累计收敛', + stabilized: (duration: string, tolerance: string) => + `在 ${duration}时已稳定至 ${tolerance} 范围内`, + notStabilized: (duration: string, tolerance: string) => + `在 ${duration}运行期内未稳定至 ${tolerance} 范围内`, + span: '稳定后的累计范围', + requests: '达到稳定时的请求数', + maxDeviation: '后续最大偏差', + method: (checkpoint: string, confirmation: string) => + `回溯性单次运行诊断：每 ${checkpoint}计算一次累计指标，并要求之后至少连续 ${confirmation}满足稳定条件；并非置信区间，也不预测复跑结果。`, rootTrajectories: '根轨迹数', kishCoverage: 'Kish 有效根轨迹覆盖数', kishExplanation: '仅表示覆盖多样性，并非统计有效样本量', - smallestWindow: '最小时间窗口', - successfulRequests: '个成功请求', }, } as const; -const observedWindowDuration = (seconds: number | undefined, locale: Locale): string | null => { - if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds <= 0) return null; +const convergenceDuration = ( + seconds: number | undefined, + locale: Locale, + adjective = false, +): string => { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds <= 0) { + return locale === 'zh' ? '未知时长' : 'unknown duration'; + } const minutes = seconds / 60; if (locale === 'zh') { return Number.isInteger(minutes) ? `${minutes} 分钟` : `${seconds} 秒`; } - return Number.isInteger(minutes) ? `${minutes}-minute` : `${seconds}-second`; + if (!Number.isInteger(minutes)) return adjective ? `${seconds}-second` : `${seconds} seconds`; + if (adjective) return `${minutes}-minute`; + return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`; }; /** - * One-run descriptive diagnostics for the x metric. This intentionally says - * what the range is not: the windows are dependent slices of one run, so the - * min/max cannot support confidence-interval or rerun-prediction language. + * Retrospective cumulative-prefix convergence for the current x metric. The + * result describes one run; it is neither a sampling confidence interval nor + * a prediction of how a rerun will behave. */ -const generateObservedWindowRangeHTML = (d: InferenceData, locale: Locale): string => { - const min = d.observedXMin; - const max = d.observedXMax; - const count = d.observed_window_count ?? 0; +const generateConvergenceHTML = (d: InferenceData, locale: Locale): string => { + if (!d.convergenceEvaluated) return ''; + + const t = CONVERGENCE_STRINGS[locale]; + const tolerance = `±${Number(((d.convergence_tolerance_ratio ?? 0) * 100).toFixed(2))}%`; + const stabilized = typeof d.convergenceTimeSeconds === 'number' && d.convergenceTimeSeconds > 0; + const status = stabilized + ? t.stabilized(convergenceDuration(d.convergenceTimeSeconds, locale), tolerance) + : t.notStabilized(convergenceDuration(d.convergence_horizon_seconds, locale, true), tolerance); + const method = t.method( + convergenceDuration(d.convergence_checkpoint_seconds, locale, true), + convergenceDuration(d.convergence_min_confirmation_seconds, locale), + ); + + const coverageRows: string[] = []; if ( - count < 2 || - typeof min !== 'number' || - !Number.isFinite(min) || - min <= 0 || - typeof max !== 'number' || - !Number.isFinite(max) || - max <= min + stabilized && + typeof d.convergenceXMin === 'number' && + Number.isFinite(d.convergenceXMin) && + typeof d.convergenceXMax === 'number' && + Number.isFinite(d.convergenceXMax) ) { - return ''; + coverageRows.push(tooltipLine(t.span, `${fmt(d.convergenceXMin)}–${fmt(d.convergenceXMax)}`)); + } + if (stabilized && typeof d.convergenceRequests === 'number') { + coverageRows.push(tooltipLine(t.requests, d.convergenceRequests)); + } + if (stabilized && typeof d.convergenceMaxRelativeDeviation === 'number') { + coverageRows.push( + tooltipLine( + t.maxDeviation, + `${Number((d.convergenceMaxRelativeDeviation * 100).toFixed(2))}%`, + ), + ); } - - const t = OBSERVED_RANGE_STRINGS[locale]; - const duration = observedWindowDuration(d.observed_window_seconds, locale); - const title = duration ? t.title(duration) : t.genericTitle; - const expected = d.observed_window_expected_count; - const windowNote = - typeof expected === 'number' && expected > count - ? t.partialWindows(count, expected) - : t.completeWindows(count); - - const coverageRows: string[] = []; if (typeof d.root_trajectory_count === 'number') { coverageRows.push(tooltipLine(t.rootTrajectories, d.root_trajectory_count)); } @@ -179,21 +200,13 @@ const generateObservedWindowRangeHTML = (d: InferenceData, locale: Locale): stri ), ); } - if (typeof d.observed_window_min_requests === 'number') { - const value = - locale === 'zh' - ? `${d.observed_window_min_requests}${t.successfulRequests}` - : `${d.observed_window_min_requests} ${t.successfulRequests}`; - coverageRows.push(tooltipLine(t.smallestWindow, value)); - } - return ` -
+
- ${title}: ${fmt(min)}–${fmt(max)} + ${t.title}: ${status}
- ${windowNote} + ${method}
${coverageRows.join('')}
`; @@ -413,7 +426,7 @@ export const generateTooltipContent = (config: TooltipConfig): string => {
` : '' } - ${generateObservedWindowRangeHTML(d, locale)} + ${generateConvergenceHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
@@ -474,7 +487,7 @@ export const generateOverlayTooltipContent = (config: OverlayTooltipConfig): str
${yLabel}: ${fmt(d.y)}
- ${generateObservedWindowRangeHTML(d, locale)} + ${generateConvergenceHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
@@ -546,7 +559,7 @@ export const generateGPUGraphTooltipContent = (config: TooltipConfig): string =>
` : '' } - ${generateObservedWindowRangeHTML(d, locale)} + ${generateConvergenceHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index ce81c909..42d4fd90 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -90,7 +90,7 @@ describe('rowToAggDataEntry', () => { expect(entry.median_intvty).toBe(12.5); }); - it('passes AgentX observed-window and root-coverage diagnostics through', () => { + it('passes AgentX window, coverage, and convergence diagnostics through', () => { const entry = rowToAggDataEntry( makeRow({ metrics: { @@ -107,6 +107,15 @@ describe('rowToAggDataEntry', () => { observed_window_p75_e2el_max: 79.2, observed_window_p90_intvty_min: 22.6, observed_window_p90_intvty_max: 30.7, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + convergence_p90_intvty_time_seconds: 1200, + convergence_p90_intvty_requests: 407, + convergence_p90_intvty_min: 43.36036, + convergence_p90_intvty_max: 46.97824, + convergence_p90_intvty_max_relative_deviation: 0.04385, } as unknown as BenchmarkRow['metrics'], }), ); @@ -121,6 +130,15 @@ describe('rowToAggDataEntry', () => { expect(entry.observed_window_p90_ttft_max).toBe(4.114); expect(entry.observed_window_p75_e2el_max).toBe(79.2); expect(entry.observed_window_p90_intvty_min).toBe(22.6); + expect(entry.convergence_checkpoint_seconds).toBe(300); + expect(entry.convergence_tolerance_ratio).toBe(0.05); + expect(entry.convergence_min_confirmation_seconds).toBe(1200); + expect(entry.convergence_horizon_seconds).toBe(3600); + expect(entry.convergence_p90_intvty_time_seconds).toBe(1200); + expect(entry.convergence_p90_intvty_requests).toBe(407); + expect(entry.convergence_p90_intvty_min).toBe(43.36036); + expect(entry.convergence_p90_intvty_max).toBe(46.97824); + expect(entry.convergence_p90_intvty_max_relative_deviation).toBe(0.04385); }); it('defaults missing metrics to 0', () => { diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index ff0d08e2..28b8254a 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -196,6 +196,40 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { observed_window_p75_intvty_max: m.observed_window_p75_intvty_max, observed_window_p90_intvty_min: m.observed_window_p90_intvty_min, observed_window_p90_intvty_max: m.observed_window_p90_intvty_max, + convergence_checkpoint_seconds: m.convergence_checkpoint_seconds, + convergence_tolerance_ratio: m.convergence_tolerance_ratio, + convergence_min_confirmation_seconds: m.convergence_min_confirmation_seconds, + convergence_horizon_seconds: m.convergence_horizon_seconds, + convergence_p75_ttft_time_seconds: m.convergence_p75_ttft_time_seconds, + convergence_p75_ttft_requests: m.convergence_p75_ttft_requests, + convergence_p75_ttft_min: m.convergence_p75_ttft_min, + convergence_p75_ttft_max: m.convergence_p75_ttft_max, + convergence_p75_ttft_max_relative_deviation: m.convergence_p75_ttft_max_relative_deviation, + convergence_p90_ttft_time_seconds: m.convergence_p90_ttft_time_seconds, + convergence_p90_ttft_requests: m.convergence_p90_ttft_requests, + convergence_p90_ttft_min: m.convergence_p90_ttft_min, + convergence_p90_ttft_max: m.convergence_p90_ttft_max, + convergence_p90_ttft_max_relative_deviation: m.convergence_p90_ttft_max_relative_deviation, + convergence_p75_e2el_time_seconds: m.convergence_p75_e2el_time_seconds, + convergence_p75_e2el_requests: m.convergence_p75_e2el_requests, + convergence_p75_e2el_min: m.convergence_p75_e2el_min, + convergence_p75_e2el_max: m.convergence_p75_e2el_max, + convergence_p75_e2el_max_relative_deviation: m.convergence_p75_e2el_max_relative_deviation, + convergence_p90_e2el_time_seconds: m.convergence_p90_e2el_time_seconds, + convergence_p90_e2el_requests: m.convergence_p90_e2el_requests, + convergence_p90_e2el_min: m.convergence_p90_e2el_min, + convergence_p90_e2el_max: m.convergence_p90_e2el_max, + convergence_p90_e2el_max_relative_deviation: m.convergence_p90_e2el_max_relative_deviation, + convergence_p75_intvty_time_seconds: m.convergence_p75_intvty_time_seconds, + convergence_p75_intvty_requests: m.convergence_p75_intvty_requests, + convergence_p75_intvty_min: m.convergence_p75_intvty_min, + convergence_p75_intvty_max: m.convergence_p75_intvty_max, + convergence_p75_intvty_max_relative_deviation: m.convergence_p75_intvty_max_relative_deviation, + convergence_p90_intvty_time_seconds: m.convergence_p90_intvty_time_seconds, + convergence_p90_intvty_requests: m.convergence_p90_intvty_requests, + convergence_p90_intvty_min: m.convergence_p90_intvty_min, + convergence_p90_intvty_max: m.convergence_p90_intvty_max, + convergence_p90_intvty_max_relative_deviation: m.convergence_p90_intvty_max_relative_deviation, }; } diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 1cab5a38..93c17a27 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -167,4 +167,41 @@ export const METRIC_KEYS = new Set([ 'observed_window_p75_intvty_max', 'observed_window_p90_intvty_min', 'observed_window_p90_intvty_max', + // AgentX retrospective cumulative-prefix convergence. A metric stabilizes + // only when every later checkpoint remains within the configured symmetric + // ratio band and the minimum confirmation horizon is available. + 'convergence_checkpoint_seconds', + 'convergence_tolerance_ratio', + 'convergence_min_confirmation_seconds', + 'convergence_horizon_seconds', + 'convergence_p75_ttft_time_seconds', + 'convergence_p75_ttft_requests', + 'convergence_p75_ttft_min', + 'convergence_p75_ttft_max', + 'convergence_p75_ttft_max_relative_deviation', + 'convergence_p90_ttft_time_seconds', + 'convergence_p90_ttft_requests', + 'convergence_p90_ttft_min', + 'convergence_p90_ttft_max', + 'convergence_p90_ttft_max_relative_deviation', + 'convergence_p75_e2el_time_seconds', + 'convergence_p75_e2el_requests', + 'convergence_p75_e2el_min', + 'convergence_p75_e2el_max', + 'convergence_p75_e2el_max_relative_deviation', + 'convergence_p90_e2el_time_seconds', + 'convergence_p90_e2el_requests', + 'convergence_p90_e2el_min', + 'convergence_p90_e2el_max', + 'convergence_p90_e2el_max_relative_deviation', + 'convergence_p75_intvty_time_seconds', + 'convergence_p75_intvty_requests', + 'convergence_p75_intvty_min', + 'convergence_p75_intvty_max', + 'convergence_p75_intvty_max_relative_deviation', + 'convergence_p90_intvty_time_seconds', + 'convergence_p90_intvty_requests', + 'convergence_p90_intvty_min', + 'convergence_p90_intvty_max', + 'convergence_p90_intvty_max_relative_deviation', ]); diff --git a/packages/db/research/agentx-stability-backfill.sql b/packages/db/research/agentx-stability-backfill.sql index 4bebfba2..a5da56f7 100644 --- a/packages/db/research/agentx-stability-backfill.sql +++ b/packages/db/research/agentx-stability-backfill.sql @@ -1,8 +1,9 @@ \set ON_ERROR_STOP on --- Idempotently backfill descriptive AgentX stability fields from the stored --- request timeline. These are observed non-overlapping-window ranges, not --- confidence intervals or rerun prediction intervals. +-- Idempotently backfill AgentX stability fields from the stored request +-- timeline. The Pareto whisker uses retrospective cumulative-prefix +-- convergence; the older non-overlapping-window ranges remain available for +-- drift inspection. Neither is a confidence interval or rerun prediction. -- -- Required psql variables: -- expected_branch_id Neon branch id this write is allowed to target @@ -45,6 +46,8 @@ WITH candidates AS ( SELECT br.id, floor((br.metrics->>'duration_seconds')::float8 / 600)::int AS expected_windows, + (floor((br.metrics->>'duration_seconds')::float8 / 300) * 300)::int + AS convergence_horizon_seconds, atr.request_timeline FROM benchmark_results br JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id @@ -125,6 +128,154 @@ WITH candidates AS ( CASE WHEN min(p90_itl) > 0 THEN 1 / min(p90_itl) END AS p90_intvty_max FROM window_metrics GROUP BY id +), prefix_metrics AS ( + SELECT + c.id, + c.convergence_horizon_seconds, + checkpoint.seconds::int AS checkpoint_seconds, + count(t.timestamp_ns)::int AS request_count, + count(*) FILTER (WHERE t.ttft_s > 0)::int AS ttft_request_count, + count(*) FILTER (WHERE t.e2el_s > 0)::int AS e2el_request_count, + count(*) FILTER (WHERE t.itl_s > 0)::int AS intvty_request_count, + percentile_cont(.75) WITHIN GROUP (ORDER BY t.ttft_s) + FILTER (WHERE t.ttft_s > 0) AS p75_ttft, + percentile_cont(.9) WITHIN GROUP (ORDER BY t.ttft_s) + FILTER (WHERE t.ttft_s > 0) AS p90_ttft, + percentile_cont(.75) WITHIN GROUP (ORDER BY t.e2el_s) + FILTER (WHERE t.e2el_s > 0) AS p75_e2el, + percentile_cont(.9) WITHIN GROUP (ORDER BY t.e2el_s) + FILTER (WHERE t.e2el_s > 0) AS p90_e2el, + percentile_cont(.75) WITHIN GROUP (ORDER BY t.itl_s) + FILTER (WHERE t.itl_s > 0) AS p75_itl, + percentile_cont(.9) WITHIN GROUP (ORDER BY t.itl_s) + FILTER (WHERE t.itl_s > 0) AS p90_itl + FROM candidates c + CROSS JOIN LATERAL generate_series( + 300, + c.convergence_horizon_seconds, + 300 + ) checkpoint(seconds) + JOIN timed t + ON t.id = c.id + AND t.timestamp_ns >= t.origin_ns + AND t.timestamp_ns < t.origin_ns + checkpoint.seconds * 1e9 + GROUP BY c.id, c.convergence_horizon_seconds, checkpoint.seconds +), prefix_values AS ( + SELECT + pm.id, + pm.convergence_horizon_seconds, + pm.checkpoint_seconds, + metric.request_count, + metric.metric_key, + metric.metric_value + FROM prefix_metrics pm + CROSS JOIN LATERAL (VALUES + ('p75_ttft', pm.p75_ttft, pm.ttft_request_count), + ('p90_ttft', pm.p90_ttft, pm.ttft_request_count), + ('p75_e2el', pm.p75_e2el, pm.e2el_request_count), + ('p90_e2el', pm.p90_e2el, pm.e2el_request_count), + ( + 'p75_intvty', + CASE WHEN pm.p75_itl > 0 THEN 1 / pm.p75_itl END, + pm.intvty_request_count + ), + ( + 'p90_intvty', + CASE WHEN pm.p90_itl > 0 THEN 1 / pm.p90_itl END, + pm.intvty_request_count + ) + ) metric(metric_key, metric_value, request_count) +), final_values AS ( + SELECT + id, + metric_key, + metric_value AS final_value + FROM prefix_values + WHERE checkpoint_seconds = convergence_horizon_seconds +), stabilization_candidates AS ( + SELECT + pv.id, + pv.metric_key, + pv.checkpoint_seconds, + pv.request_count + FROM prefix_values pv + JOIN final_values fv USING (id, metric_key) + WHERE pv.convergence_horizon_seconds - pv.checkpoint_seconds >= 1200 + AND pv.metric_value > 0 + AND fv.final_value > 0 + AND NOT EXISTS ( + SELECT 1 + FROM prefix_values later + WHERE later.id = pv.id + AND later.metric_key = pv.metric_key + AND later.checkpoint_seconds >= pv.checkpoint_seconds + AND ( + later.metric_value IS NULL + OR later.metric_value <= 0 + OR abs(ln(later.metric_value / fv.final_value)) + > ln(1.05::float8) + 1e-12 + ) + ) +), stabilization AS ( + SELECT DISTINCT ON (id, metric_key) + id, + metric_key, + checkpoint_seconds AS time_seconds, + request_count + FROM stabilization_candidates + ORDER BY id, metric_key, checkpoint_seconds +), convergence_summary AS ( + SELECT + s.id, + s.metric_key, + s.time_seconds, + s.request_count, + min(pv.metric_value) AS min_value, + max(pv.metric_value) AS max_value, + max(abs(pv.metric_value / fv.final_value - 1)) AS max_relative_deviation + FROM stabilization s + JOIN prefix_values pv + ON pv.id = s.id + AND pv.metric_key = s.metric_key + AND pv.checkpoint_seconds >= s.time_seconds + JOIN final_values fv + ON fv.id = s.id + AND fv.metric_key = s.metric_key + GROUP BY s.id, s.metric_key, s.time_seconds, s.request_count +), convergence_kv AS ( + SELECT + id, + 'convergence_' || metric_key || '_time_seconds' AS key, + to_jsonb(time_seconds) AS value + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_requests', + to_jsonb(request_count) + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_min', + to_jsonb(round(min_value::numeric, 5)) + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_max', + to_jsonb(round(max_value::numeric, 5)) + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_max_relative_deviation', + to_jsonb(round(max_relative_deviation::numeric, 5)) + FROM convergence_summary +), convergence_patches AS ( + SELECT id, jsonb_object_agg(key, value) AS patch + FROM convergence_kv + GROUP BY id ), root_counts AS ( SELECT id, root_id, count(*)::bigint AS request_count FROM raw @@ -164,10 +315,16 @@ WITH candidates AS ( 'observed_window_p75_intvty_min', round(ws.p75_intvty_min::numeric, 5), 'observed_window_p75_intvty_max', round(ws.p75_intvty_max::numeric, 5), 'observed_window_p90_intvty_min', round(ws.p90_intvty_min::numeric, 5), - 'observed_window_p90_intvty_max', round(ws.p90_intvty_max::numeric, 5) - )) AS patch + 'observed_window_p90_intvty_max', round(ws.p90_intvty_max::numeric, 5), + 'convergence_checkpoint_seconds', 300, + 'convergence_tolerance_ratio', 0.05, + 'convergence_min_confirmation_seconds', 1200, + 'convergence_horizon_seconds', c.convergence_horizon_seconds + )) || COALESCE(cp.patch, '{}'::jsonb) AS patch FROM window_summary ws JOIN root_summary rs USING (id) + JOIN candidates c USING (id) + LEFT JOIN convergence_patches cp USING (id) ) SELECT id, patch FROM patches; diff --git a/packages/db/src/etl/agentic-v3-flatten.ts b/packages/db/src/etl/agentic-v3-flatten.ts index 80656094..c9e3b6b1 100644 --- a/packages/db/src/etl/agentic-v3-flatten.ts +++ b/packages/db/src/etl/agentic-v3-flatten.ts @@ -78,6 +78,22 @@ const V3_SCALAR_PATHS: [string[], string][] = [ ['request_metrics', 'stability', 'root_trajectory_largest_share'], 'root_trajectory_largest_share', ], + [ + ['request_metrics', 'stability', 'convergence', 'checkpoint_seconds'], + 'convergence_checkpoint_seconds', + ], + [ + ['request_metrics', 'stability', 'convergence', 'tolerance_ratio'], + 'convergence_tolerance_ratio', + ], + [ + ['request_metrics', 'stability', 'convergence', 'min_confirmation_seconds'], + 'convergence_min_confirmation_seconds', + ], + [ + ['request_metrics', 'stability', 'convergence', 'horizon_seconds'], + 'convergence_horizon_seconds', + ], // Deliberately NOT mapped (yet): cache.overall/prefix_cache_hits/queries, // kv_cache.cpu_*, tokens.prompt_by_source, sources[] — new v3 detail we don't // consume anywhere; add here + METRIC_KEYS when a view needs them. @@ -162,5 +178,26 @@ export function flattenAgenticAggRow(row: Record): Record = {}): Record { expect(m.observed_window_p90_ttft_max).toBeCloseTo(4.114, 3); expect(m.observed_window_p75_e2el_max).toBeCloseTo(79.2, 3); expect(m.observed_window_p90_intvty_min).toBeCloseTo(22.621, 3); + // retrospective cumulative-prefix convergence + expect(m.convergence_checkpoint_seconds).toBe(300); + expect(m.convergence_tolerance_ratio).toBe(0.05); + expect(m.convergence_min_confirmation_seconds).toBe(1200); + expect(m.convergence_horizon_seconds).toBe(3600); + expect(m.convergence_p75_ttft_time_seconds).toBe(900); + expect(m.convergence_p75_ttft_requests).toBe(300); + expect(m.convergence_p75_ttft_min).toBeCloseTo(1.21, 5); + expect(m.convergence_p75_ttft_max_relative_deviation).toBeCloseTo(0.031, 5); + expect(m.convergence_p90_intvty_time_seconds).toBe(1200); + expect(m.convergence_p90_intvty_requests).toBe(407); + expect(m.convergence_p90_intvty_min).toBeCloseTo(43.36036, 5); + expect(m.convergence_p90_intvty_max).toBeCloseTo(46.97824, 5); + expect(m.convergence_p90_intvty_max_relative_deviation).toBeCloseTo(0.04385, 5); + expect(m).not.toHaveProperty('convergence_p90_ttft_time_seconds'); // nested containers must not leak into metrics expect(m).not.toHaveProperty('request_metrics'); expect(m).not.toHaveProperty('server_metrics');