diff --git a/packages/app/cypress/component/scatter-graph.cy.tsx b/packages/app/cypress/component/scatter-graph.cy.tsx index 692f6a77..4772d79e 100644 --- a/packages/app/cypress/component/scatter-graph.cy.tsx +++ b/packages/app/cypress/component/scatter-graph.cy.tsx @@ -732,6 +732,12 @@ describe('ScatterGraph', () => { y: 260 - index * 40, precision: Precision.FP4, run_url: runUrl, + convergenceXMin: x * 0.96, + convergenceXMax: x * 1.04, + convergenceTimeSeconds: 1200, + convergenceRequests: 407, + convergenceMaxRelativeDeviation: 0.04, + convergence_tolerance_ratio: 0.05, }), ), hardwareConfig: hwConfig, @@ -777,10 +783,16 @@ describe('ScatterGraph', () => { cy.get('#test-scatter-dismiss-preview svg .unofficial-overlay-pt').should('have.length', 3); cy.get('#test-scatter-dismiss-preview svg .overlay-roofline-path').should('exist'); + cy.get('#test-scatter-dismiss-preview svg .convergence-range[data-source="overlay"]').should( + 'exist', + ); cy.get('[data-testid="dismiss-preview"]').click(); cy.get('#test-scatter-dismiss-preview svg .unofficial-overlay-pt').should('not.exist'); cy.get('#test-scatter-dismiss-preview svg .overlay-roofline-path').should('not.exist'); + cy.get('#test-scatter-dismiss-preview svg .convergence-range[data-source="overlay"]').should( + 'not.exist', + ); }); }); diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 1ce282c2..c3b6b8f4 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -41,9 +41,15 @@ const percentileLadder = (prefix: string, base: number): Record [`std_${prefix}`]: base * 0.3, }); -const agenticMetrics = (conc: number): Record => { +const agenticMetrics = (conc: number, stabilized = true): Record => { const scale = conc / 16; const itl = 0.011 * scale; + const p75Ttft = 0.4 * scale * 1.2; + const p90Ttft = 0.4 * scale * 1.5; + const p75E2e = 8 * scale * 1.2; + const p90E2e = 8 * scale * 1.5; + const p75Interactivity = 1 / (itl * 1.2); + const p90Interactivity = 1 / (itl * 1.5); return { ...percentileLadder('ttft', 0.4 * scale), ...percentileLadder('tpot', 0.012 * scale), @@ -58,6 +64,63 @@ const agenticMetrics = (conc: number): Record => { output_tput_per_gpu: 210, input_tput_per_gpu: 740, total_tput_tps: 7600 * conc * 0.05, + observed_window_seconds: 600, + observed_window_expected_count: 6, + observed_window_count: 6, + observed_window_min_requests: Math.max(12, Math.round(conc * 1.25)), + root_trajectory_count: Math.max(3, Math.round(conc / 4)), + root_trajectory_kish_effective_count: Math.max(1.8, conc / 8), + root_trajectory_largest_share: Math.min(0.56, 8 / conc), + observed_window_p75_ttft_min: p75Ttft * 0.55, + observed_window_p75_ttft_max: p75Ttft * 1.45, + observed_window_p90_ttft_min: p90Ttft * 0.5, + observed_window_p90_ttft_max: p90Ttft * 1.5, + observed_window_p75_e2el_min: p75E2e * 0.75, + observed_window_p75_e2el_max: p75E2e * 1.25, + observed_window_p90_e2el_min: p90E2e * 0.7, + observed_window_p90_e2el_max: p90E2e * 1.3, + observed_window_p75_intvty_min: p75Interactivity * 0.72, + observed_window_p75_intvty_max: p75Interactivity * 1.28, + observed_window_p90_intvty_min: p90Interactivity * 0.68, + observed_window_p90_intvty_max: p90Interactivity * 1.32, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + ...(stabilized + ? { + convergence_p75_ttft_time_seconds: 900, + convergence_p75_ttft_requests: Math.round(conc * 25), + convergence_p75_ttft_min: p75Ttft * 0.97, + convergence_p75_ttft_max: p75Ttft * 1.03, + convergence_p75_ttft_max_relative_deviation: 0.03, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: Math.round(conc * 25.4), + convergence_p90_ttft_min: p90Ttft * 0.96, + convergence_p90_ttft_max: p90Ttft * 1.04, + convergence_p90_ttft_max_relative_deviation: 0.04, + convergence_p75_e2el_time_seconds: 900, + convergence_p75_e2el_requests: Math.round(conc * 25), + convergence_p75_e2el_min: p75E2e * 0.97, + convergence_p75_e2el_max: p75E2e * 1.03, + convergence_p75_e2el_max_relative_deviation: 0.03, + convergence_p90_e2el_time_seconds: 1200, + convergence_p90_e2el_requests: Math.round(conc * 25.4), + convergence_p90_e2el_min: p90E2e * 0.96, + convergence_p90_e2el_max: p90E2e * 1.04, + convergence_p90_e2el_max_relative_deviation: 0.04, + convergence_p75_intvty_time_seconds: 900, + convergence_p75_intvty_requests: Math.round(conc * 25), + convergence_p75_intvty_min: p75Interactivity * 0.97, + convergence_p75_intvty_max: p75Interactivity * 1.03, + convergence_p75_intvty_max_relative_deviation: 0.03, + convergence_p90_intvty_time_seconds: 1200, + convergence_p90_intvty_requests: Math.round(conc * 25.4), + convergence_p90_intvty_min: p90Interactivity * 0.96, + convergence_p90_intvty_max: p90Interactivity * 1.04, + convergence_p90_intvty_max_relative_deviation: 0.04, + } + : {}), }; }; @@ -116,7 +179,7 @@ const agenticBenchmarks = agenticGpus.flatMap((g) => offload_mode: 'off', benchmark_type: 'agentic_traces', image: 'vllm/vllm-openai:v0.9.0', - metrics: agenticMetrics(conc), + metrics: agenticMetrics(conc, !(g.hardware === 'b200' && conc === 16)), workers: null, date: AGENTIC_DATE, run_url: null, @@ -131,12 +194,21 @@ const fixedSequenceBenchmarks = agenticBenchmarks.map((row, index) => ({ benchmark_type: 'single_turn', })); +const interceptDashboardMetadata = () => { + cy.intercept('GET', '/api/v1/workflow-info*', { + body: { runs: [], changelogs: [], configs: [], runConfigs: [] }, + }).as('workflowInfo'); + cy.intercept('GET', '/api/v1/trace-availability*', { body: {} }).as('traceAvailability'); +}; + const interceptAgenticData = () => { + interceptDashboardMetadata(); cy.intercept('GET', '/api/v1/availability', { body: agenticAvailability }).as('availability'); cy.intercept('GET', '/api/v1/benchmarks*', { body: agenticBenchmarks }).as('benchmarks'); }; const interceptFixedSequenceData = () => { + interceptDashboardMetadata(); cy.intercept('GET', '/api/v1/availability', { body: agenticAvailability }).as('availability'); cy.intercept('GET', '/api/v1/benchmarks*', { body: fixedSequenceBenchmarks }).as('benchmarks'); }; @@ -191,6 +263,90 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Time To First Token'); }); + it('renders post-stabilization spans with an explicit non-inferential caveat', () => { + cy.get( + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="official"]', + ).should('have.length.greaterThan', 0); + cy.get('[data-testid="convergence-range-key"]') + .should('contain.text', 'Cumulative span after ±5% stabilization') + .and('contain.text', 'retrospective') + .and('contain.text', 'not a confidence interval'); + + cy.get('[data-testid="scatter-graph"] svg .dot-group').then(($dots) => { + const stabilizedDot = [...$dots].find( + (dot) => + (dot as unknown as SVGGElement & { __data__: { id?: number } }).__data__.id === 900001, + ); + expect(stabilizedDot, 'stabilized fixture point').not.to.equal(undefined); + cy.wrap(stabilizedDot).find('.visible-shape').click({ force: true }); + }); + cy.get('[data-chart-tooltip]:visible') + .should('contain.text', 'Stabilized by 20 minutes at ±5%') + .and('contain.text', 'Post-stabilization span') + .and('contain.text', 'Maximum later deviation') + .and('contain.text', 'Retrospective within-run diagnostic') + .and('contain.text', 'not a confidence interval or rerun prediction') + .and('contain.text', 'Kish-effective root coverage'); + }); + + it('renders no whisker when the cumulative estimate did not stabilize', () => { + cy.get( + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="official"]', + ).then(($ranges) => { + const hasUnstablePoint = [...$ranges].some( + (range) => + (range as unknown as SVGGElement & { __data__: { point: { id?: number } } }).__data__ + .point.id === 900000, + ); + expect(hasUnstablePoint).to.equal(false); + }); + + cy.get('[data-testid="scatter-graph"] svg .dot-group').then(($dots) => { + const unstableDot = [...$dots].find( + (dot) => + (dot as unknown as SVGGElement & { __data__: { id?: number } }).__data__.id === 900000, + ); + expect(unstableDot, 'non-stabilized fixture point').not.to.equal(undefined); + cy.wrap(unstableDot).find('.visible-shape').click({ force: true }); + }); + cy.get('[data-chart-tooltip]:visible') + .should('contain.text', 'Not stabilized within the 60-minute run at ±5%') + .and('not.contain.text', 'Post-stabilization span'); + }); + + it('keeps convergence ranges anchored through chart zoom', () => { + cy.get( + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="official"] .convergence-range-stem', + ) + .first() + .then(($stem) => { + const before = [$stem.attr('x1'), $stem.attr('x2')]; + cy.get('[data-testid="scatter-graph"] svg') + .first() + .then(($svg) => { + const svg = $svg[0]; + const bounds = svg.getBoundingClientRect(); + svg.dispatchEvent( + new WheelEvent('wheel', { + deltaY: -240, + clientX: bounds.x + bounds.width / 2, + clientY: bounds.y + bounds.height / 2, + shiftKey: true, + bubbles: true, + cancelable: true, + }), + ); + }); + cy.wait(300); + cy.wrap($stem).should(($zoomedStem) => { + expect( + [$zoomedStem.attr('x1'), $zoomedStem.attr('x2')], + 'range geometry after zoom', + ).not.to.deep.equal(before); + }); + }); + }); + it('switches the x-axis to E2E Latency and updates the heading', () => { cy.get('[data-testid="x-axis-mode-e2e"]').click(); cy.get('[data-testid="x-axis-mode-e2e"]').should('have.attr', 'aria-selected', 'true'); @@ -371,6 +527,11 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () }); expect(total).to.be.greaterThan(0); }); + cy.get( + '[data-testid="inference-chart-display"] [data-testid="convergence-range"][data-source="overlay"]', + ) + .should('have.length.greaterThan', 0) + .and('have.attr', 'data-run-index', '0'); }); it('normalized-e2e mode shows suppression banner for unofficial-run overlays', () => { diff --git a/packages/app/src/components/inference/hooks/useChartData.test.ts b/packages/app/src/components/inference/hooks/useChartData.test.ts index c4998add..497c3a07 100644 --- a/packages/app/src/components/inference/hooks/useChartData.test.ts +++ b/packages/app/src/components/inference/hooks/useChartData.test.ts @@ -5,8 +5,11 @@ import { dedupeRowsToLatestPerConfig, filterByGPU, flipRooflineDirection, + remapChartPoint, } from './useChartData'; +import type { InferenceData } from '@/components/inference/types'; + interface DedupeInput { id: number; hardware: string; @@ -151,3 +154,25 @@ describe('flipRooflineDirection', () => { } }); }); + +describe('remapChartPoint', () => { + it('preserves the selected metric roof flag while flattening its y value', () => { + const point = { + x: 1, + y: 2, + hwKey: 'h100', + date: '2026-06-01', + tp: 100, + conc: 1, + precision: 'fp8', + p90_ttft: 3, + tpPerGpu: { y: 42, roof: true }, + } as InferenceData; + + expect(remapChartPoint(point, 'tpPerGpu', 'p90_ttft')).toMatchObject({ + x: 3, + y: 42, + roof: true, + }); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index 197fcd6b..31732a7a 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -29,6 +29,7 @@ import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { calculateCostsForGpus, calculatePowerForGpus } from '@/lib/utils'; import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; +import { withConvergenceRange } from '@/components/inference/utils/convergence-range'; import { applyQuickFilters, computeAvailableQuickFilters, @@ -139,6 +140,32 @@ export function flipRooflineDirection(dir: RooflineDirection): RooflineDirection return FLIP_MAP[dir]; } +/** + * Remap a transformed benchmark point onto the currently selected chart axes. + * Keep the selected y-metric's roofline marker on the flattened point: chart + * consumers read this marker after the metric object has been reduced to `y`. + */ +export function remapChartPoint( + point: InferenceData, + metricKey: YAxisMetricKey, + xAxisField: keyof AggDataEntry, + isOnE2eFrontier?: boolean, +): InferenceData { + const metric = point[metricKey] as { y: number; roof: boolean } | undefined; + const xCandidate = (point as Partial)[xAxisField]; + + return withConvergenceRange( + { + ...point, + x: typeof xCandidate === 'number' ? xCandidate : point.x, + y: metric?.y ?? point.y, + roof: metric?.roof ?? false, + isOnE2eFrontier, + }, + String(xAxisField), + ); +} + /** The dedup key fields a chart series is identified by. */ interface DedupeRow { hardware: string; @@ -526,25 +553,11 @@ export function useChartData( ? filteredData .filter((d) => metricKey in d) .map((d: InferenceData) => { - const yValue = (d[metricKey] as { y: number })?.y ?? d.y; - const roof = (d[metricKey] as { roof: boolean })?.roof ?? false; - // xAxisField is `keyof AggDataEntry`; InferenceData embeds those - // fields via `Partial>`, so a typed - // accessor catches a future field rename (silent fallthrough to - // d.x would otherwise mask the regression). - const xCandidate = (d as Partial)[xAxisField]; - const xValue = typeof xCandidate === 'number' ? xCandidate : d.x; const isOnE2eFrontier = e2eParetoSet === null ? undefined : isPersistedBenchmarkId(d.id) && e2eParetoSet.has(d.id); - return { - ...d, - x: xValue, - y: yValue, - roof, - isOnE2eFrontier, - }; + return remapChartPoint(d, metricKey, xAxisField, isOnE2eFrontier); }) // When TTFT is on the x-axis, apply the latency limit to filter // overload outliers (fixed-seq conc=2048 rows with TTFT > 60s that diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8b45e0f7..f1f6d2d3 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -219,6 +219,70 @@ export interface AggDataEntry { total_prompt_tokens?: number; /** Total generated (output) tokens. */ total_generation_tokens?: number; + /** Width of each non-overlapping descriptive window, in seconds. */ + observed_window_seconds?: number; + /** Configured number of complete windows in the measurement duration. */ + observed_window_expected_count?: number; + /** Number of those windows containing at least one successful request. */ + observed_window_count?: number; + /** Smallest successful-request count among observed windows. */ + observed_window_min_requests?: number; + /** Distinct root source trajectories represented by all requests. */ + root_trajectory_count?: number; + /** Kish diversity count from root-trajectory request shares (not a statistical ESS). */ + root_trajectory_kish_effective_count?: number; + /** Fraction of requests belonging to the most represented root trajectory. */ + root_trajectory_largest_share?: number; + observed_window_p75_ttft_min?: number; + observed_window_p75_ttft_max?: number; + observed_window_p90_ttft_min?: number; + observed_window_p90_ttft_max?: number; + observed_window_p75_e2el_min?: number; + observed_window_p75_e2el_max?: number; + observed_window_p90_e2el_min?: number; + observed_window_p90_e2el_max?: number; + observed_window_p75_intvty_min?: number; + observed_window_p75_intvty_max?: number; + observed_window_p90_intvty_min?: number; + observed_window_p90_intvty_max?: number; + /** Width of each cumulative convergence checkpoint, in seconds. */ + convergence_checkpoint_seconds?: number; + /** Symmetric multiplicative tolerance around the final cumulative estimate. */ + convergence_tolerance_ratio?: number; + /** Required duration after stabilization, in seconds. */ + convergence_min_confirmation_seconds?: number; + /** Last complete cumulative checkpoint included in the diagnostic. */ + convergence_horizon_seconds?: number; + convergence_p75_ttft_time_seconds?: number; + convergence_p75_ttft_requests?: number; + convergence_p75_ttft_min?: number; + convergence_p75_ttft_max?: number; + convergence_p75_ttft_max_relative_deviation?: number; + convergence_p90_ttft_time_seconds?: number; + convergence_p90_ttft_requests?: number; + convergence_p90_ttft_min?: number; + convergence_p90_ttft_max?: number; + convergence_p90_ttft_max_relative_deviation?: number; + convergence_p75_e2el_time_seconds?: number; + convergence_p75_e2el_requests?: number; + convergence_p75_e2el_min?: number; + convergence_p75_e2el_max?: number; + convergence_p75_e2el_max_relative_deviation?: number; + convergence_p90_e2el_time_seconds?: number; + convergence_p90_e2el_requests?: number; + convergence_p90_e2el_min?: number; + convergence_p90_e2el_max?: number; + convergence_p90_e2el_max_relative_deviation?: number; + convergence_p75_intvty_time_seconds?: number; + convergence_p75_intvty_requests?: number; + convergence_p75_intvty_min?: number; + convergence_p75_intvty_max?: number; + convergence_p75_intvty_max_relative_deviation?: number; + convergence_p90_intvty_time_seconds?: number; + convergence_p90_intvty_requests?: number; + convergence_p90_intvty_min?: number; + convergence_p90_intvty_max?: number; + convergence_p90_intvty_max_relative_deviation?: number; } /** @@ -243,7 +307,17 @@ export interface InferenceData extends Partial ({ hwKey, precision, x, y, tp, conc: 16, framework: 'vllm' }) as unknown as InferenceData; +const convergedPoint = ( + hwKey: string, + x: number, + y: number, + min: number, + max: number, +): InferenceData => ({ + ...point(hwKey, 'fp8', x, y, 8), + convergenceXMin: min, + convergenceXMax: max, + convergenceTimeSeconds: 1200, + convergenceRequests: 407, + convergenceMaxRelativeDeviation: 0.04, + convergence_tolerance_ratio: 0.05, +}); + // h100 owns both axis extremes so hiding b200 / showing fp4 keeps the niced // domains identical — exactly the toggle case that must not rebuild. const POINTS: InferenceData[] = [ @@ -139,6 +156,7 @@ function baseOverlayState() { // ── Harness ────────────────────────────────────────────────────────────────── function mountChart(props?: Partial[0]>) { let forceUpdate: () => void = noop; + let currentProps = props ?? {}; function Harness() { // ScatterGraph and D3Chart are React.memo'd; mocked context hooks bypass // React's context subscription, so re-renders are driven through a @@ -154,7 +172,7 @@ function mountChart(props?: Partial[0]>) { chartDefinition: CHART_DEFINITION, transitionDuration: 0, caption: `v${version}`, - ...props, + ...currentProps, }); } @@ -167,6 +185,10 @@ function mountChart(props?: Partial[0]>) { return { container, rerender: () => act(() => forceUpdate()), + updateProps: (next: Partial[0]>) => { + currentProps = { ...currentProps, ...next }; + act(() => forceUpdate()); + }, unmount: () => { act(() => root.unmount()); container.remove(); @@ -215,6 +237,64 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('renders convergence ranges only for Pareto-front points and fits their x extent', () => { + const data = [ + convergedPoint('h100', 1, 50, 0.5, 1.8), + convergedPoint('h100', 2, 100, 1.2, 8), + // Below the running upper-right envelope, so it must not get a whisker. + convergedPoint('h100', 1.5, 40, 0.8, 6), + ]; + const { container, unmount } = mountChart({ + data, + chartDefinition: { + ...CHART_DEFINITION, + y_roofline: 'upper_right', + } as unknown as ChartDefinition, + }); + + const ranges = [ + ...container.querySelectorAll('.convergence-range[data-source="official"]'), + ]; + expect(ranges).toHaveLength(2); + expect(ranges.every((range) => range.style.pointerEvents === 'none')).toBe(true); + expect( + ranges.some( + (range) => + (range as SVGGElement & { __data__: { point: InferenceData } }).__data__.point.x === 1.5, + ), + ).toBe(false); + + const wideRange = ranges.find( + (range) => + (range as SVGGElement & { __data__: { point: InferenceData } }).__data__.point.x === 2, + )!; + const stem = wideRange.querySelector('.convergence-range-stem')!; + const pointDot = dotGroups(container).find( + (dot) => (dot as SVGGElement & { __data__: InferenceData }).__data__.x === 2, + )!; + const pointX = Number( + pointDot.getAttribute('transform')?.match(/translate\((?[^,]+)/u)?.groups?.x, + ); + expect(Number(stem.getAttribute('x2'))).toBeGreaterThan(pointX); + // The convergence max participates in the domain, so its cap remains inside + // the 800px chart rather than being clipped at the right edge. + expect(Number(stem.getAttribute('x2'))).toBeLessThan(800); + unmount(); + }); + + it('renders no whisker for a point that was evaluated but did not stabilize', () => { + const unstable = { + ...point('h100', 'fp8', 2, 100, 8), + convergenceEvaluated: true, + convergence_tolerance_ratio: 0.05, + convergence_horizon_seconds: 3600, + }; + const { container, unmount } = mountChart({ data: [unstable] }); + + expect(container.querySelectorAll('.convergence-range')).toHaveLength(0); + unmount(); + }); + it('hides a toggled-off hw via opacity without rebuilding the chart', () => { const { container, rerender, unmount } = mountChart(); const buildsAfterMount = rebuildCount(); @@ -377,6 +457,47 @@ describe('ScatterGraph toggle decoration', () => { unmount(); }); + it('renders and removes overlay convergence ranges through the overlay path', () => { + const overlayPoints = [ + convergedPoint('h100', 30, 250, 20, 36), + convergedPoint('h100', 35, 300, 28, 48), + ].map((p) => ({ + ...p, + run_url: 'https://github.com/o/r/actions/runs/123', + })); + overlayState.current = { + ...baseOverlayState(), + isUnofficialRun: true, + activeOverlayHwTypes: new Set(['h100']), + allOverlayHwTypes: new Set(['h100']), + runIndexByUrl: { 'https://github.com/o/r/actions/runs/123': 0 }, + unofficialRunInfos: [ + { id: '123', branch: 'test-branch', url: 'https://github.com/o/r/actions/runs/123' }, + ], + }; + const overlayData = { + data: overlayPoints, + hardwareConfig: HARDWARE_CONFIG, + } as unknown as Parameters[0]['overlayData']; + const { container, updateProps, unmount } = mountChart({ + overlayData, + chartDefinition: { + ...CHART_DEFINITION, + y_roofline: 'upper_right', + } as unknown as ChartDefinition, + }); + + const ranges = [ + ...container.querySelectorAll('.convergence-range[data-source="overlay"]'), + ]; + expect(ranges).toHaveLength(2); + expect(ranges.every((range) => range.getAttribute('stroke') === overlayRunColor(0))).toBe(true); + + updateProps({ overlayData: undefined }); + expect(container.querySelectorAll('.convergence-range[data-source="overlay"]')).toHaveLength(0); + unmount(); + }); + it('applies quick filters to unofficial-run overlay markers', () => { const overlayPoints = [point('h100', 'fp8', 30, 300, 2), point('h100', 'fp8', 35, 350, 4)].map( (p) => ({ ...p, run_url: 'https://github.com/o/r/actions/runs/123' }), diff --git a/packages/app/src/components/inference/ui/ScatterGraph.tsx b/packages/app/src/components/inference/ui/ScatterGraph.tsx index 16e3ab6d..51eff7d3 100644 --- a/packages/app/src/components/inference/ui/ScatterGraph.tsx +++ b/packages/app/src/components/inference/ui/ScatterGraph.tsx @@ -261,6 +261,41 @@ interface ScaleConfigValue { nice: boolean; _isLog?: boolean; } + +interface ConvergenceRangeMark { + key: string; + point: InferenceData & { convergenceXMin: number; convergenceXMax: number }; + hw: string; + precision: string; + source: 'official' | 'overlay'; + color: string; + visible: boolean; + runIndex?: number; +} + +const hasConvergenceRange = ( + point: InferenceData, +): point is InferenceData & { convergenceXMin: number; convergenceXMax: number } => + typeof point.convergenceTimeSeconds === 'number' && + Number.isFinite(point.convergenceTimeSeconds) && + point.convergenceTimeSeconds > 0 && + typeof point.convergenceXMin === 'number' && + Number.isFinite(point.convergenceXMin) && + point.convergenceXMin > 0 && + typeof point.convergenceXMax === 'number' && + Number.isFinite(point.convergenceXMax) && + point.convergenceXMax > point.convergenceXMin; + +const convergenceRangeLegendLabel = ( + locale: 'en' | 'zh', + toleranceRatio: number | undefined, +): string => { + const tolerance = + typeof toleranceRatio === 'number' ? `±${Number((toleranceRatio * 100).toFixed(2))}%` : ''; + return locale === 'zh' + ? `累计指标在${tolerance ? ` ${tolerance} ` : ''}内稳定后的范围——回溯性诊断，并非置信区间` + : `Cumulative span after ${tolerance ? `${tolerance} ` : ''}stabilization — retrospective, not a confidence interval`; +}; const isSameScaleConfig = (a: ScaleConfigValue, b: ScaleConfigValue): boolean => a.type === b.type && a.nice === b.nice && @@ -935,6 +970,48 @@ const ScatterGraph = React.memo( [allPointLabelsByKey], ); + // Window ranges are a descriptive annotation on the Pareto frontier, not + // additional observations and not an alternate frontier. Keeping this + // list sourced from `rooflines` / `overlayRooflines` guarantees a + // dominated scatter point can never stretch the axis merely because it + // carries diagnostics. + const visibleFrontierPoints = useMemo(() => { + const points: InferenceData[] = []; + for (const [key, front] of Object.entries(rooflines)) { + const hw = key.split('_').slice(0, -1).join('_'); + const precision = key.split('_').pop()!; + if (effectiveActiveHwTypes.has(hw) && selectedPrecisions.includes(precision)) { + points.push(...front); + } + } + for (const group of Object.values(overlayRooflines)) { + if (activeOverlayHwTypes.has(group.hwKey)) points.push(...group.points); + } + return points; + }, [ + rooflines, + overlayRooflines, + effectiveActiveHwTypes, + selectedPrecisions, + activeOverlayHwTypes, + ]); + + const convergenceRangePoints = useMemo( + () => visibleFrontierPoints.filter(hasConvergenceRange), + [visibleFrontierPoints], + ); + + // A mixed-policy comparison gets a generic key rather than claiming one + // tolerance applies to every point. + const convergenceToleranceRatio = useMemo(() => { + const tolerances = new Set( + convergenceRangePoints + .map((point) => point.convergence_tolerance_ratio) + .filter((ratio): ratio is number => typeof ratio === 'number' && Number.isFinite(ratio)), + ); + return tolerances.size === 1 ? [...tolerances][0] : undefined; + }, [convergenceRangePoints]); + // Ref for trackedConfigIds (needs to be current at event time inside D3 handlers) const trackedConfigIdsRef = useRef(trackedConfigIds); trackedConfigIdsRef.current = trackedConfigIds; @@ -966,7 +1043,13 @@ const ScatterGraph = React.memo( const ext = xExtentOverride ?? (visiblePoints.length > 0 - ? (d3.extent(visiblePoints, (d) => d.x) as [number, number]) + ? (d3.extent([ + ...visiblePoints.map((point) => point.x), + ...convergenceRangePoints.flatMap((point) => [ + point.convergenceXMin, + point.convergenceXMax, + ]), + ]) as [number, number]) : ([0, 100] as [number, number])); let useLog = false; @@ -986,7 +1069,15 @@ const ScatterGraph = React.memo( nice: niceAxes, _isLog: useLog, }; - }, [visiblePoints, isInputTputMetric, xLabel, scaleType, niceAxes, xExtentOverride]); + }, [ + visiblePoints, + convergenceRangePoints, + isInputTputMetric, + xLabel, + scaleType, + niceAxes, + xExtentOverride, + ]); const xScaleConfig = useStableValue(xScaleConfigRaw, isSameScaleConfig); const yScaleConfigRaw = useMemo(() => { @@ -1059,6 +1150,20 @@ const ScatterGraph = React.memo( [effectiveActiveHwTypes, selectedPrecisions], ); + const convergenceRangeOpacity = useCallback( + (el: SVGGElement): number => { + const hw = el.dataset.hwKey; + const precision = el.dataset.precision; + if (!hw || !precision) return 0; + const active = + el.dataset.source === 'overlay' + ? activeOverlayHwTypes.has(hw) + : effectiveActiveHwTypes.has(hw); + return active && selectedPrecisions.includes(precision) ? 0.42 : 0; + }, + [activeOverlayHwTypes, effectiveActiveHwTypes, selectedPrecisions], + ); + // --- Interaction state ref --- // Latest visibility predicates, color resolvers, and active sets — read by // long-lived D3 closures (layer renders, zoom handlers, hover handlers). @@ -1115,13 +1220,20 @@ const ScatterGraph = React.memo( if (!isRooflineVisible(this)) return 0; return this.dataset.hwKey === hwKey ? null : '0.15'; }); + root + .selectAll('.convergence-range[data-source="official"]') + .style('opacity', function () { + const base = convergenceRangeOpacity(this); + if (base === 0) return 0; + return this.dataset.hwKey === hwKey ? base : 0.06; + }); root .selectAll('.parallelism-label, .line-label') .style('opacity', function () { return labelOpacityForHover((this as SVGGElement).dataset, hwKey); }); }, - [isPointVisible, isRooflineVisible], + [isPointVisible, isRooflineVisible, convergenceRangeOpacity], ); const handleLegendHoverEnd = useCallback(() => { @@ -1134,6 +1246,9 @@ const ScatterGraph = React.memo( root.selectAll('.roofline-path').style('opacity', function () { return isRooflineVisible(this) ? 1 : 0; }); + root.selectAll('.convergence-range').style('opacity', function () { + return convergenceRangeOpacity(this); + }); root .selectAll('.parallelism-label, .line-label') .style('opacity', function () { @@ -1143,7 +1258,13 @@ const ScatterGraph = React.memo( selectedPrecisions, ); }); - }, [isPointVisible, isRooflineVisible, effectiveActiveHwTypes, selectedPrecisions]); + }, [ + isPointVisible, + isRooflineVisible, + convergenceRangeOpacity, + effectiveActiveHwTypes, + selectedPrecisions, + ]); // --- Zoom config --- const eventPrefix = chartDefinition.chartType === 'e2e' ? 'latency' : 'interactivity'; @@ -1403,6 +1524,105 @@ const ScatterGraph = React.memo( .style('transition', 'opacity 150ms ease') .style('opacity', (d) => (d.visible ? 1 : 0)); + // Thin, capped horizontal whiskers summarize cumulative estimates + // after retrospective stabilization. They deliberately live in the + // roofline layer (behind points), carry no pointer events, and are + // never used to compute the Pareto frontier. + const convergenceRangeMarks: ConvergenceRangeMark[] = []; + for (const [key, front] of Object.entries(rooflines)) { + const hw = key.split('_').slice(0, -1).join('_'); + const precision = key.split('_').pop()!; + const visible = + ir.effectiveActiveHwTypes.has(hw) && ir.selectedPrecisions.includes(precision); + front.forEach((point, index) => { + if (!hasConvergenceRange(point)) return; + convergenceRangeMarks.push({ + key: `official-${key}-${point.date}-${point.id ?? 'na'}-${point.conc}-${point.x}-${point.y}-${index}`, + point, + hw, + precision, + source: 'official', + color: ir.getCssColor(ir.resolveColor(hw)), + visible, + }); + }); + } + if (overlayData) { + for (const [key, group] of Object.entries(overlayRooflines)) { + if (!overlayData.hardwareConfig[group.hwKey]) continue; + group.points.forEach((point, index) => { + if (!hasConvergenceRange(point)) return; + convergenceRangeMarks.push({ + key: `overlay-${key}-${point.date}-${point.id ?? 'na'}-${point.conc}-${point.x}-${point.y}-${index}`, + point, + hw: group.hwKey, + precision: point.precision, + source: 'overlay', + color: overlayRunColor(group.runIndex), + visible: + ir.activeOverlayHwTypes.has(group.hwKey) && + ir.selectedPrecisions.includes(point.precision), + runIndex: group.runIndex, + }); + }); + } + } + + const convergenceRanges = rooflinesLayer + .selectAll('.convergence-range') + .data(convergenceRangeMarks, (mark) => mark.key) + .join( + (enter) => { + const group = enter.append('g').attr('class', 'convergence-range'); + group.append('line').attr('class', 'convergence-range-stem'); + group.append('line').attr('class', 'convergence-range-cap-min'); + group.append('line').attr('class', 'convergence-range-cap-max'); + return group; + }, + (update) => update, + (exit) => exit.remove(), + ) + .attr('data-testid', 'convergence-range') + .attr('data-source', (mark) => mark.source) + .attr('data-hw-key', (mark) => mark.hw) + .attr('data-precision', (mark) => mark.precision) + .attr('data-run-index', (mark) => mark.runIndex ?? null) + .attr('aria-hidden', 'true') + .attr('fill', 'none') + .attr('stroke', (mark) => mark.color) + .attr('stroke-width', 1) + .attr('stroke-linecap', 'round') + .style('pointer-events', 'none') + .style('transition', 'opacity 150ms ease') + .style('opacity', (mark) => (mark.visible ? 0.42 : 0)); + + convergenceRanges.selectAll('line').attr('vector-effect', 'non-scaling-stroke'); + convergenceRanges.each(function (mark) { + const group = d3.select(this); + const minX = xScale(mark.point.convergenceXMin); + const maxX = xScale(mark.point.convergenceXMax); + const y = yScale(mark.point.y); + group + .select('.convergence-range-stem') + .attr('x1', minX) + .attr('x2', maxX) + .attr('y1', y) + .attr('y2', y); + group + .select('.convergence-range-cap-min') + .attr('x1', minX) + .attr('x2', minX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + group + .select('.convergence-range-cap-max') + .attr('x1', maxX) + .attr('x2', maxX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + }); + convergenceRanges.lower(); + // Parallelism labels interface LabelSeg { segKey: string; @@ -1854,6 +2074,35 @@ const ScatterGraph = React.memo( } }); + // Keep convergence whiskers anchored to the full-run point while + // zooming. End caps stay a fixed seven CSS pixels tall. + zoomGroup + .selectAll('.convergence-range') + .each(function (mark) { + const group = d3.select(this); + const minX = newXScale(mark.point.convergenceXMin); + const maxX = newXScale(mark.point.convergenceXMax); + const y = newYScale(mark.point.y); + group + .select('.convergence-range-stem') + .attr('x1', minX) + .attr('x2', maxX) + .attr('y1', y) + .attr('y2', y); + group + .select('.convergence-range-cap-min') + .attr('x1', minX) + .attr('x2', minX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + group + .select('.convergence-range-cap-max') + .attr('x1', maxX) + .attr('x2', maxX) + .attr('y1', y - 3.5) + .attr('y2', y + 3.5); + }); + // Update gradient coordinates if (showGradientLabels) { Object.entries(allPointLabelsByKey).forEach(([key, pointLabels]) => { @@ -2683,6 +2932,16 @@ const ScatterGraph = React.memo( } }); + // Convergence ranges follow the same visibility/recolor path as + // their frontier series without touching their zoomed geometry. + zoomGroup.selectAll('.convergence-range').each(function () { + const el = d3.select(this); + el.style('opacity', convergenceRangeOpacity(this)); + if (this.dataset.source === 'official' && this.dataset.hwKey) { + el.attr('stroke', ir.getCssColor(ir.resolveColor(this.dataset.hwKey))); + } + }); + // Parallelism / line labels: visibility via data attributes (mirrors // handleLegendHoverEnd). Placement-level updates happen below. zoomGroup @@ -2740,6 +2999,7 @@ const ScatterGraph = React.memo( showGradientLabels, showLineLabels, gradientColorByPoint, + convergenceRangeOpacity, ]); // D3 custom layers are keyed additions, so removing the overlay layer from @@ -2749,7 +3009,11 @@ const ScatterGraph = React.memo( if (overlayData) return; const svg = chartRef.current?.getSvgElement?.(); if (!svg) return; - d3.select(svg).selectAll('.unofficial-overlay-pt, .overlay-roofline-path').remove(); + d3.select(svg) + .selectAll( + '.unofficial-overlay-pt, .overlay-roofline-path, .convergence-range[data-source="overlay"]', + ) + .remove(); }, [overlayData]); // Dismiss tooltip on filter changes @@ -2937,6 +3201,32 @@ const ScatterGraph = React.memo( })), ]} disableActiveSort={false} + keyIndicators={ + convergenceRangePoints.length > 0 ? ( +
+ + {convergenceRangeLegendLabel(locale, convergenceToleranceRatio)} +
+ ) : undefined + } isLegendExpanded={isLegendExpanded} onExpandedChange={(expanded) => { setIsLegendExpanded(expanded); diff --git a/packages/app/src/components/inference/utils.test.ts b/packages/app/src/components/inference/utils.test.ts index 35e48ad2..394b4489 100644 --- a/packages/app/src/components/inference/utils.test.ts +++ b/packages/app/src/components/inference/utils.test.ts @@ -233,6 +233,30 @@ describe('processOverlayChartData', () => { expect(result[0].x).toBe(0.12); }); + it('stamps convergence bounds matching the overlay x-axis metric', () => { + const data = [ + pt({ + tpPerGpu: { y: 42, roof: false }, + p90_ttft: 2, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: 407, + convergence_p90_ttft_min: 1.4, + convergence_p90_ttft_max: 1.52, + convergence_p90_ttft_max_relative_deviation: 0.043, + } as any), + ]; + const result = processOverlayChartData(data, 'e2e', 'y_tpPerGpu', 'p90_ttft'); + + expect(result[0].convergenceXMin).toBe(1.4); + expect(result[0].convergenceXMax).toBe(1.52); + expect(result[0].convergenceTimeSeconds).toBe(1200); + expect(result[0].convergenceRequests).toBe(407); + }); + it('filters e2e TTFT outliers exceeding y_latency_limit', () => { const data = [ pt({ tpPerGpu: { y: 10, roof: false }, p90_ttft: 0.5, median_e2el: 1 } as any), diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index 2b8074d9..8df884cc 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -7,6 +7,7 @@ import chartDefinitions from '@/components/inference/inference-chart-config.json'; import { e2eFrontierWinners } from '@/components/inference/utils/e2eFrontier'; import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisField'; +import { withConvergenceRange } from '@/components/inference/utils/convergence-range'; import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; @@ -127,7 +128,7 @@ export function processOverlayChartData( .map((d: InferenceData) => { const yValue = (d[metricKey] as { y: number })?.y ?? d.y; const xValue = (d as any)[xAxisField] ?? d.x; - return { ...d, x: xValue, y: yValue }; + return withConvergenceRange({ ...d, x: xValue, y: yValue }, xAxisField); }) .filter( (d) => !isTtftX || isAgentic || !chartDef.y_latency_limit || d.x <= chartDef.y_latency_limit, diff --git a/packages/app/src/components/inference/utils/convergence-range.test.ts b/packages/app/src/components/inference/utils/convergence-range.test.ts new file mode 100644 index 00000000..1aa9c9ab --- /dev/null +++ b/packages/app/src/components/inference/utils/convergence-range.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; + +import type { InferenceData } from '@/components/inference/types'; + +import { + convergenceEvaluatedForXAxis, + convergenceRangeForXAxis, + withConvergenceRange, +} from './convergence-range'; + +const diagnostic = { + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, +}; + +describe('convergenceRangeForXAxis', () => { + it('selects the stabilization result matching the resolved x-axis field', () => { + const point = { + ...diagnostic, + convergence_p90_ttft_time_seconds: 900, + convergence_p90_ttft_requests: 312, + convergence_p90_ttft_min: 1.4, + convergence_p90_ttft_max: 1.52, + convergence_p90_ttft_max_relative_deviation: 0.041, + convergence_p75_e2el_time_seconds: 1200, + convergence_p75_e2el_requests: 407, + convergence_p75_e2el_min: 40, + convergence_p75_e2el_max: 42, + convergence_p75_e2el_max_relative_deviation: 0.03, + }; + + expect(convergenceRangeForXAxis(point, 'p90_ttft')).toEqual({ + min: 1.4, + max: 1.52, + timeSeconds: 900, + requests: 312, + maxRelativeDeviation: 0.041, + }); + expect(convergenceRangeForXAxis(point, 'p75_e2el')?.timeSeconds).toBe(1200); + }); + + it('distinguishes an evaluated metric that did not stabilize from missing diagnostics', () => { + expect(convergenceEvaluatedForXAxis(diagnostic, 'p90_intvty')).toBe(true); + expect(convergenceRangeForXAxis(diagnostic, 'p90_intvty')).toBeNull(); + expect(convergenceEvaluatedForXAxis({}, 'p90_intvty')).toBe(false); + expect(convergenceEvaluatedForXAxis(diagnostic, 'median_intvty')).toBe(false); + }); + + it('rejects invalid bounds and metadata while allowing a flat stabilized span', () => { + const base = { + ...diagnostic, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: 407, + convergence_p90_ttft_min: 2, + convergence_p90_ttft_max: 2, + convergence_p90_ttft_max_relative_deviation: 0, + }; + expect(convergenceRangeForXAxis(base, 'p90_ttft')).toMatchObject({ min: 2, max: 2 }); + expect( + convergenceRangeForXAxis({ ...base, convergence_p90_ttft_min: 0 }, 'p90_ttft'), + ).toBeNull(); + expect( + convergenceRangeForXAxis({ ...base, convergence_p90_ttft_max: 1 }, 'p90_ttft'), + ).toBeNull(); + expect( + convergenceRangeForXAxis( + { ...base, convergence_p90_ttft_max_relative_deviation: Number.NaN }, + 'p90_ttft', + ), + ).toBeNull(); + }); +}); + +describe('withConvergenceRange', () => { + it('stamps chart-ready convergence metadata and clears it when the x field changes', () => { + const point = { + x: 2, + y: 100, + date: '2026-07-21', + tp: 8, + conc: 16, + precision: 'fp4', + hwKey: 'b300-vllm', + tpPerGpu: { y: 100, roof: true }, + tpPerMw: { y: 1, roof: true }, + costh: { y: 1, roof: true }, + costn: { y: 1, roof: true }, + costr: { y: 1, roof: true }, + costhi: { y: 1, roof: true }, + costni: { y: 1, roof: true }, + costri: { y: 1, roof: true }, + ...diagnostic, + convergence_p90_ttft_time_seconds: 1200, + convergence_p90_ttft_requests: 407, + convergence_p90_ttft_min: 1, + convergence_p90_ttft_max: 3, + convergence_p90_ttft_max_relative_deviation: 0.04, + } as InferenceData; + + const stamped = withConvergenceRange(point, 'p90_ttft'); + expect(stamped.convergenceEvaluated).toBe(true); + expect(stamped.convergenceXMin).toBe(1); + expect(stamped.convergenceXMax).toBe(3); + expect(stamped.convergenceTimeSeconds).toBe(1200); + expect(stamped.convergenceRequests).toBe(407); + + const cleared = withConvergenceRange(stamped, 'median_ttft'); + expect(cleared.convergenceEvaluated).toBe(false); + expect(cleared.convergenceXMin).toBeUndefined(); + expect(cleared.convergenceTimeSeconds).toBeUndefined(); + }); +}); diff --git a/packages/app/src/components/inference/utils/convergence-range.ts b/packages/app/src/components/inference/utils/convergence-range.ts new file mode 100644 index 00000000..b642a8b4 --- /dev/null +++ b/packages/app/src/components/inference/utils/convergence-range.ts @@ -0,0 +1,78 @@ +import type { AggDataEntry, InferenceData } from '@/components/inference/types'; + +export interface ConvergenceRange { + min: number; + max: number; + timeSeconds: number; + requests: number; + maxRelativeDeviation: number; +} + +const CONVERGENCE_X_FIELD = /^(?:p75|p90)_(?:ttft|e2el|intvty)$/u; + +const finiteNumber = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value); + +/** Whether this run evaluated cumulative convergence for the selected x field. */ +export function convergenceEvaluatedForXAxis( + point: Partial, + xAxisField: string, +): boolean { + return ( + CONVERGENCE_X_FIELD.test(xAxisField) && + finiteNumber(point.convergence_checkpoint_seconds) && + point.convergence_checkpoint_seconds > 0 && + finiteNumber(point.convergence_tolerance_ratio) && + point.convergence_tolerance_ratio >= 0 && + finiteNumber(point.convergence_min_confirmation_seconds) && + point.convergence_min_confirmation_seconds >= 0 && + finiteNumber(point.convergence_horizon_seconds) && + point.convergence_horizon_seconds > 0 + ); +} + +/** Resolve post-stabilization cumulative bounds for the chart's current x field. */ +export function convergenceRangeForXAxis( + point: Partial, + xAxisField: string, +): ConvergenceRange | null { + if (!convergenceEvaluatedForXAxis(point, xAxisField)) return null; + + const values = point as unknown as Record; + const prefix = `convergence_${xAxisField}`; + const min = values[`${prefix}_min`]; + const max = values[`${prefix}_max`]; + const timeSeconds = values[`${prefix}_time_seconds`]; + const requests = values[`${prefix}_requests`]; + const maxRelativeDeviation = values[`${prefix}_max_relative_deviation`]; + if ( + !finiteNumber(min) || + min <= 0 || + !finiteNumber(max) || + max < min || + !finiteNumber(timeSeconds) || + timeSeconds <= 0 || + !finiteNumber(requests) || + requests < 0 || + !finiteNumber(maxRelativeDeviation) || + maxRelativeDeviation < 0 + ) { + return null; + } + + return { min, max, timeSeconds, requests, maxRelativeDeviation }; +} + +/** Stamp the selected convergence result onto a chart-ready point. */ +export function withConvergenceRange(point: InferenceData, xAxisField: string): InferenceData { + const range = convergenceRangeForXAxis(point, xAxisField); + return { + ...point, + convergenceEvaluated: convergenceEvaluatedForXAxis(point, xAxisField), + convergenceXMin: range?.min, + convergenceXMax: range?.max, + convergenceTimeSeconds: range?.timeSeconds, + convergenceRequests: range?.requests, + convergenceMaxRelativeDeviation: range?.maxRelativeDeviation, + }; +} diff --git a/packages/app/src/components/inference/utils/tooltip-utils.test.ts b/packages/app/src/components/inference/utils/tooltip-utils.test.ts index 893819f5..b7e319be 100644 --- a/packages/app/src/components/inference/utils/tooltip-utils.test.ts +++ b/packages/app/src/components/inference/utils/tooltip-utils.test.ts @@ -71,6 +71,25 @@ function tooltipConfig(overrides: Partial = {}): TooltipConfig { }; } +function convergencePoint(overrides: Partial = {}): InferenceData { + return pt({ + benchmark_type: 'agentic_traces', + convergenceEvaluated: true, + convergenceXMin: 43.36036, + convergenceXMax: 46.97824, + convergenceTimeSeconds: 1200, + convergenceRequests: 407, + convergenceMaxRelativeDeviation: 0.04385, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + root_trajectory_count: 3, + root_trajectory_kish_effective_count: 1.69, + ...overrides, + }); +} + // =========================================================================== // getPointLabel // =========================================================================== @@ -425,6 +444,60 @@ describe('generateTooltipContent', () => { expect(html).toContain('Track Over Time'); expect(html).not.toContain('Untrack Over Time'); }); + + it('explains cumulative stabilization without making inferential claims', () => { + const html = generateTooltipContent(tooltipConfig({ data: convergencePoint() })); + + expect(html).toContain( + 'Cumulative convergence: Stabilized by 20 minutes at ±5%', + ); + expect(html).toContain('Post-stabilization span: 43.36–46.978'); + expect(html).toContain('Requests at stabilization: 407'); + expect(html).toContain('Maximum later deviation: 4.38%'); + expect(html).toContain('5-minute cumulative checkpoints'); + expect(html).toContain('at least 20 minutes of later confirmation'); + expect(html).toContain('not a confidence interval or rerun prediction'); + expect(html).toContain('Root trajectories: 3'); + expect(html).toContain('Kish-effective root coverage: 1.69'); + expect(html).toContain('not a statistical effective sample size'); + }); + + it('renders natural Chinese convergence language', () => { + const html = generateTooltipContent(tooltipConfig({ locale: 'zh', data: convergencePoint() })); + + expect(html).toContain('累计收敛: 在 20 分钟时已稳定至 ±5% 范围内'); + expect(html).toContain('稳定后的累计范围: 43.36–46.978'); + expect(html).toContain('达到稳定时的请求数: 407'); + expect(html).toContain('后续最大偏差: 4.38%'); + expect(html).toContain('回溯性单次运行诊断'); + expect(html).toContain('并非置信区间，也不预测复跑结果'); + expect(html).toContain('根轨迹数: 3'); + expect(html).toContain('Kish 有效根轨迹覆盖数: 1.69'); + expect(html).toContain('并非统计有效样本量'); + }); + + it('marks an evaluated metric that never stabilized and shows no span', () => { + const html = generateTooltipContent( + tooltipConfig({ + data: convergencePoint({ + convergenceTimeSeconds: undefined, + convergenceXMin: undefined, + convergenceXMax: undefined, + convergenceRequests: undefined, + convergenceMaxRelativeDeviation: undefined, + }), + }), + ); + expect(html).toContain('Not stabilized within the 60-minute run at ±5%'); + expect(html).not.toContain('Post-stabilization span'); + }); + + it('omits convergence copy when the selected metric was not evaluated', () => { + const html = generateTooltipContent( + tooltipConfig({ data: convergencePoint({ convergenceEvaluated: false }) }), + ); + expect(html).not.toContain('convergence-diagnostic-tooltip'); + }); }); // =========================================================================== @@ -506,6 +579,15 @@ describe('generateOverlayTooltipContent', () => { expect(html).not.toContain('Offload Type'); expect(html).not.toContain('CPU Cache Hit Rate'); }); + + it('shows the same convergence caveat for unofficial overlays', () => { + const html = generateOverlayTooltipContent(overlayConfig({ data: convergencePoint() })); + + expect(html).toContain('Stabilized by 20 minutes at ±5%'); + expect(html).toContain('Post-stabilization span'); + expect(html).toContain('not a confidence interval or rerun prediction'); + expect(html).toContain('Kish-effective root coverage'); + }); }); // =========================================================================== diff --git a/packages/app/src/components/inference/utils/tooltipUtils.ts b/packages/app/src/components/inference/utils/tooltipUtils.ts index 9226381d..3758cfc6 100644 --- a/packages/app/src/components/inference/utils/tooltipUtils.ts +++ b/packages/app/src/components/inference/utils/tooltipUtils.ts @@ -99,6 +99,119 @@ export const fmt = (v: number): string => { return String(rounded); }; +const CONVERGENCE_STRINGS = { + en: { + title: 'Cumulative convergence', + stabilized: (duration: string, tolerance: string) => + `Stabilized by ${duration} at ${tolerance}`, + notStabilized: (duration: string, tolerance: string) => + `Not stabilized within the ${duration} run at ${tolerance}`, + span: 'Post-stabilization span', + requests: 'Requests at stabilization', + maxDeviation: 'Maximum later deviation', + method: (checkpoint: string, confirmation: string) => + `Retrospective within-run diagnostic using ${checkpoint} cumulative checkpoints and at least ${confirmation} of later confirmation; not a confidence interval or rerun prediction.`, + rootTrajectories: 'Root trajectories', + kishCoverage: 'Kish-effective root coverage', + kishExplanation: 'coverage diversity, not a statistical effective sample size', + }, + zh: { + title: '累计收敛', + stabilized: (duration: string, tolerance: string) => + `在 ${duration}时已稳定至 ${tolerance} 范围内`, + notStabilized: (duration: string, tolerance: string) => + `在 ${duration}运行期内未稳定至 ${tolerance} 范围内`, + span: '稳定后的累计范围', + requests: '达到稳定时的请求数', + maxDeviation: '后续最大偏差', + method: (checkpoint: string, confirmation: string) => + `回溯性单次运行诊断：每 ${checkpoint}计算一次累计指标，并要求之后至少连续 ${confirmation}满足稳定条件；并非置信区间，也不预测复跑结果。`, + rootTrajectories: '根轨迹数', + kishCoverage: 'Kish 有效根轨迹覆盖数', + kishExplanation: '仅表示覆盖多样性，并非统计有效样本量', + }, +} as const; + +const convergenceDuration = ( + seconds: number | undefined, + locale: Locale, + adjective = false, +): string => { + if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds <= 0) { + return locale === 'zh' ? '未知时长' : 'unknown duration'; + } + const minutes = seconds / 60; + if (locale === 'zh') { + return Number.isInteger(minutes) ? `${minutes} 分钟` : `${seconds} 秒`; + } + if (!Number.isInteger(minutes)) return adjective ? `${seconds}-second` : `${seconds} seconds`; + if (adjective) return `${minutes}-minute`; + return `${minutes} ${minutes === 1 ? 'minute' : 'minutes'}`; +}; + +/** + * Retrospective cumulative-prefix convergence for the current x metric. The + * result describes one run; it is neither a sampling confidence interval nor + * a prediction of how a rerun will behave. + */ +const generateConvergenceHTML = (d: InferenceData, locale: Locale): string => { + if (!d.convergenceEvaluated) return ''; + + const t = CONVERGENCE_STRINGS[locale]; + const tolerance = `±${Number(((d.convergence_tolerance_ratio ?? 0) * 100).toFixed(2))}%`; + const stabilized = typeof d.convergenceTimeSeconds === 'number' && d.convergenceTimeSeconds > 0; + const status = stabilized + ? t.stabilized(convergenceDuration(d.convergenceTimeSeconds, locale), tolerance) + : t.notStabilized(convergenceDuration(d.convergence_horizon_seconds, locale, true), tolerance); + const method = t.method( + convergenceDuration(d.convergence_checkpoint_seconds, locale, true), + convergenceDuration(d.convergence_min_confirmation_seconds, locale), + ); + + const coverageRows: string[] = []; + if ( + stabilized && + typeof d.convergenceXMin === 'number' && + Number.isFinite(d.convergenceXMin) && + typeof d.convergenceXMax === 'number' && + Number.isFinite(d.convergenceXMax) + ) { + coverageRows.push(tooltipLine(t.span, `${fmt(d.convergenceXMin)}–${fmt(d.convergenceXMax)}`)); + } + if (stabilized && typeof d.convergenceRequests === 'number') { + coverageRows.push(tooltipLine(t.requests, d.convergenceRequests)); + } + if (stabilized && typeof d.convergenceMaxRelativeDeviation === 'number') { + coverageRows.push( + tooltipLine( + t.maxDeviation, + `${Number((d.convergenceMaxRelativeDeviation * 100).toFixed(2))}%`, + ), + ); + } + if (typeof d.root_trajectory_count === 'number') { + coverageRows.push(tooltipLine(t.rootTrajectories, d.root_trajectory_count)); + } + if (typeof d.root_trajectory_kish_effective_count === 'number') { + coverageRows.push( + tooltipLine( + t.kishCoverage, + `${fmt(d.root_trajectory_kish_effective_count)} (${t.kishExplanation})`, + ), + ); + } + return ` +
+
+ ${t.title}: ${status} +
+
+ ${method} +
+ ${coverageRows.join('')} +
`; +}; + const CACHE_STRINGS = { en: { offloadType: 'Offload Type', @@ -313,6 +426,7 @@ export const generateTooltipContent = (config: TooltipConfig): string => { ` : '' } + ${generateConvergenceHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
@@ -373,6 +487,7 @@ export const generateOverlayTooltipContent = (config: OverlayTooltipConfig): str
${yLabel}: ${fmt(d.y)}
+ ${generateConvergenceHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
@@ -444,6 +559,7 @@ export const generateGPUGraphTooltipContent = (config: TooltipConfig): string =>
` : '' } + ${generateConvergenceHTML(d, locale)} ${tooltipLine('Total GPUs', d.tp)} ${generateParallelismHTML(d)}
diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 22a16af3..42d4fd90 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -90,6 +90,57 @@ describe('rowToAggDataEntry', () => { expect(entry.median_intvty).toBe(12.5); }); + it('passes AgentX window, coverage, and convergence diagnostics through', () => { + const entry = rowToAggDataEntry( + makeRow({ + metrics: { + observed_window_seconds: 600, + observed_window_expected_count: 6, + observed_window_count: 6, + observed_window_min_requests: 21, + root_trajectory_count: 3, + root_trajectory_kish_effective_count: 1.69, + root_trajectory_largest_share: 0.74, + observed_window_p90_ttft_min: 1.416, + observed_window_p90_ttft_max: 4.114, + observed_window_p75_e2el_min: 42.1, + observed_window_p75_e2el_max: 79.2, + observed_window_p90_intvty_min: 22.6, + observed_window_p90_intvty_max: 30.7, + convergence_checkpoint_seconds: 300, + convergence_tolerance_ratio: 0.05, + convergence_min_confirmation_seconds: 1200, + convergence_horizon_seconds: 3600, + convergence_p90_intvty_time_seconds: 1200, + convergence_p90_intvty_requests: 407, + convergence_p90_intvty_min: 43.36036, + convergence_p90_intvty_max: 46.97824, + convergence_p90_intvty_max_relative_deviation: 0.04385, + } as unknown as BenchmarkRow['metrics'], + }), + ); + + expect(entry.observed_window_seconds).toBe(600); + expect(entry.observed_window_count).toBe(6); + expect(entry.observed_window_min_requests).toBe(21); + expect(entry.root_trajectory_count).toBe(3); + expect(entry.root_trajectory_kish_effective_count).toBe(1.69); + expect(entry.root_trajectory_largest_share).toBe(0.74); + expect(entry.observed_window_p90_ttft_min).toBe(1.416); + expect(entry.observed_window_p90_ttft_max).toBe(4.114); + expect(entry.observed_window_p75_e2el_max).toBe(79.2); + expect(entry.observed_window_p90_intvty_min).toBe(22.6); + expect(entry.convergence_checkpoint_seconds).toBe(300); + expect(entry.convergence_tolerance_ratio).toBe(0.05); + expect(entry.convergence_min_confirmation_seconds).toBe(1200); + expect(entry.convergence_horizon_seconds).toBe(3600); + expect(entry.convergence_p90_intvty_time_seconds).toBe(1200); + expect(entry.convergence_p90_intvty_requests).toBe(407); + expect(entry.convergence_p90_intvty_min).toBe(43.36036); + expect(entry.convergence_p90_intvty_max).toBe(46.97824); + expect(entry.convergence_p90_intvty_max_relative_deviation).toBe(0.04385); + }); + it('defaults missing metrics to 0', () => { const entry = rowToAggDataEntry(makeRow({ metrics: {} })); expect(entry.tput_per_gpu).toBe(0); diff --git a/packages/app/src/lib/benchmark-transform.ts b/packages/app/src/lib/benchmark-transform.ts index 6180b87e..28b8254a 100644 --- a/packages/app/src/lib/benchmark-transform.ts +++ b/packages/app/src/lib/benchmark-transform.ts @@ -177,6 +177,59 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { num_requests_successful: m.num_requests_successful, total_prompt_tokens: m.total_prompt_tokens, total_generation_tokens: m.total_generation_tokens, + observed_window_seconds: m.observed_window_seconds, + observed_window_expected_count: m.observed_window_expected_count, + observed_window_count: m.observed_window_count, + observed_window_min_requests: m.observed_window_min_requests, + root_trajectory_count: m.root_trajectory_count, + root_trajectory_kish_effective_count: m.root_trajectory_kish_effective_count, + root_trajectory_largest_share: m.root_trajectory_largest_share, + observed_window_p75_ttft_min: m.observed_window_p75_ttft_min, + observed_window_p75_ttft_max: m.observed_window_p75_ttft_max, + observed_window_p90_ttft_min: m.observed_window_p90_ttft_min, + observed_window_p90_ttft_max: m.observed_window_p90_ttft_max, + observed_window_p75_e2el_min: m.observed_window_p75_e2el_min, + observed_window_p75_e2el_max: m.observed_window_p75_e2el_max, + observed_window_p90_e2el_min: m.observed_window_p90_e2el_min, + observed_window_p90_e2el_max: m.observed_window_p90_e2el_max, + observed_window_p75_intvty_min: m.observed_window_p75_intvty_min, + observed_window_p75_intvty_max: m.observed_window_p75_intvty_max, + observed_window_p90_intvty_min: m.observed_window_p90_intvty_min, + observed_window_p90_intvty_max: m.observed_window_p90_intvty_max, + convergence_checkpoint_seconds: m.convergence_checkpoint_seconds, + convergence_tolerance_ratio: m.convergence_tolerance_ratio, + convergence_min_confirmation_seconds: m.convergence_min_confirmation_seconds, + convergence_horizon_seconds: m.convergence_horizon_seconds, + convergence_p75_ttft_time_seconds: m.convergence_p75_ttft_time_seconds, + convergence_p75_ttft_requests: m.convergence_p75_ttft_requests, + convergence_p75_ttft_min: m.convergence_p75_ttft_min, + convergence_p75_ttft_max: m.convergence_p75_ttft_max, + convergence_p75_ttft_max_relative_deviation: m.convergence_p75_ttft_max_relative_deviation, + convergence_p90_ttft_time_seconds: m.convergence_p90_ttft_time_seconds, + convergence_p90_ttft_requests: m.convergence_p90_ttft_requests, + convergence_p90_ttft_min: m.convergence_p90_ttft_min, + convergence_p90_ttft_max: m.convergence_p90_ttft_max, + convergence_p90_ttft_max_relative_deviation: m.convergence_p90_ttft_max_relative_deviation, + convergence_p75_e2el_time_seconds: m.convergence_p75_e2el_time_seconds, + convergence_p75_e2el_requests: m.convergence_p75_e2el_requests, + convergence_p75_e2el_min: m.convergence_p75_e2el_min, + convergence_p75_e2el_max: m.convergence_p75_e2el_max, + convergence_p75_e2el_max_relative_deviation: m.convergence_p75_e2el_max_relative_deviation, + convergence_p90_e2el_time_seconds: m.convergence_p90_e2el_time_seconds, + convergence_p90_e2el_requests: m.convergence_p90_e2el_requests, + convergence_p90_e2el_min: m.convergence_p90_e2el_min, + convergence_p90_e2el_max: m.convergence_p90_e2el_max, + convergence_p90_e2el_max_relative_deviation: m.convergence_p90_e2el_max_relative_deviation, + convergence_p75_intvty_time_seconds: m.convergence_p75_intvty_time_seconds, + convergence_p75_intvty_requests: m.convergence_p75_intvty_requests, + convergence_p75_intvty_min: m.convergence_p75_intvty_min, + convergence_p75_intvty_max: m.convergence_p75_intvty_max, + convergence_p75_intvty_max_relative_deviation: m.convergence_p75_intvty_max_relative_deviation, + convergence_p90_intvty_time_seconds: m.convergence_p90_intvty_time_seconds, + convergence_p90_intvty_requests: m.convergence_p90_intvty_requests, + convergence_p90_intvty_min: m.convergence_p90_intvty_min, + convergence_p90_intvty_max: m.convergence_p90_intvty_max, + convergence_p90_intvty_max_relative_deviation: m.convergence_p90_intvty_max_relative_deviation, }; } diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 914eed4b..93c17a27 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -145,4 +145,63 @@ export const METRIC_KEYS = new Set([ 'peak_temp_c', 'avg_util_pct', 'avg_mem_used_mb', + // AgentX realized-workload coverage and non-overlapping time-window ranges. + // Bounds describe one observed run and must never be labeled as confidence + // intervals or rerun-prediction intervals. + 'observed_window_seconds', + 'observed_window_expected_count', + 'observed_window_count', + 'observed_window_min_requests', + 'root_trajectory_count', + 'root_trajectory_kish_effective_count', + 'root_trajectory_largest_share', + 'observed_window_p75_ttft_min', + 'observed_window_p75_ttft_max', + 'observed_window_p90_ttft_min', + 'observed_window_p90_ttft_max', + 'observed_window_p75_e2el_min', + 'observed_window_p75_e2el_max', + 'observed_window_p90_e2el_min', + 'observed_window_p90_e2el_max', + 'observed_window_p75_intvty_min', + 'observed_window_p75_intvty_max', + 'observed_window_p90_intvty_min', + 'observed_window_p90_intvty_max', + // AgentX retrospective cumulative-prefix convergence. A metric stabilizes + // only when every later checkpoint remains within the configured symmetric + // ratio band and the minimum confirmation horizon is available. + 'convergence_checkpoint_seconds', + 'convergence_tolerance_ratio', + 'convergence_min_confirmation_seconds', + 'convergence_horizon_seconds', + 'convergence_p75_ttft_time_seconds', + 'convergence_p75_ttft_requests', + 'convergence_p75_ttft_min', + 'convergence_p75_ttft_max', + 'convergence_p75_ttft_max_relative_deviation', + 'convergence_p90_ttft_time_seconds', + 'convergence_p90_ttft_requests', + 'convergence_p90_ttft_min', + 'convergence_p90_ttft_max', + 'convergence_p90_ttft_max_relative_deviation', + 'convergence_p75_e2el_time_seconds', + 'convergence_p75_e2el_requests', + 'convergence_p75_e2el_min', + 'convergence_p75_e2el_max', + 'convergence_p75_e2el_max_relative_deviation', + 'convergence_p90_e2el_time_seconds', + 'convergence_p90_e2el_requests', + 'convergence_p90_e2el_min', + 'convergence_p90_e2el_max', + 'convergence_p90_e2el_max_relative_deviation', + 'convergence_p75_intvty_time_seconds', + 'convergence_p75_intvty_requests', + 'convergence_p75_intvty_min', + 'convergence_p75_intvty_max', + 'convergence_p75_intvty_max_relative_deviation', + 'convergence_p90_intvty_time_seconds', + 'convergence_p90_intvty_requests', + 'convergence_p90_intvty_min', + 'convergence_p90_intvty_max', + 'convergence_p90_intvty_max_relative_deviation', ]); diff --git a/packages/db/research/agentx-paired-signatures.sql b/packages/db/research/agentx-paired-signatures.sql new file mode 100644 index 00000000..125e4241 --- /dev/null +++ b/packages/db/research/agentx-paired-signatures.sql @@ -0,0 +1,65 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +-- Workload-matched request signatures for two benchmark rows. This is a +-- sensitivity diagnostic, not an uncertainty interval: repeated signatures +-- are generally sparse within one fixed-duration run. +COPY ( +WITH raw AS ( + SELECT + br.id, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS source_trace_id, + COALESCE(NULLIF(r->>'srcOuter', ''), NULLIF(r->>'ti', ''), '0')::int AS source_outer_idx, + COALESCE(NULLIF(r->>'srcInner', ''), '-1')::int AS source_inner_idx, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + ((r->>'end')::float8 - (r->>'start')::float8) / 1e9 AS e2e_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS tpot_s, + NULLIF(r->>'isl', '')::float8 AS isl, + NULLIF(r->>'osl', '')::float8 AS osl + FROM benchmark_results br + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + CROSS JOIN LATERAL jsonb_array_elements(atr.request_timeline->'requests') r + WHERE br.id IN (:id_a, :id_b) + AND r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), signatures AS ( + SELECT + id, + source_trace_id, + source_outer_idx, + source_inner_idx, + count(*)::int AS n, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS median_ttft_s, + percentile_cont(.5) WITHIN GROUP (ORDER BY e2e_s) AS median_e2e_s, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS median_tpot_s, + percentile_cont(.5) WITHIN GROUP (ORDER BY isl) + FILTER (WHERE isl IS NOT NULL) AS median_isl, + percentile_cont(.5) WITHIN GROUP (ORDER BY osl) + FILTER (WHERE osl IS NOT NULL) AS median_osl + FROM raw + GROUP BY id, source_trace_id, source_outer_idx, source_inner_idx +) +SELECT + a.source_trace_id, + a.source_outer_idx, + a.source_inner_idx, + a.n AS n_a, + b.n AS n_b, + a.median_ttft_s AS ttft_a, + b.median_ttft_s AS ttft_b, + a.median_e2e_s AS e2e_a, + b.median_e2e_s AS e2e_b, + a.median_tpot_s AS tpot_a, + b.median_tpot_s AS tpot_b, + a.median_isl AS isl_a, + b.median_isl AS isl_b, + a.median_osl AS osl_a, + b.median_osl AS osl_b +FROM signatures a +JOIN signatures b USING (source_trace_id, source_outer_idx, source_inner_idx) +WHERE a.id = :id_a + AND b.id = :id_b +ORDER BY a.source_trace_id, a.source_outer_idx, a.source_inner_idx +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/research/agentx-stability-backfill.sql b/packages/db/research/agentx-stability-backfill.sql new file mode 100644 index 00000000..a5da56f7 --- /dev/null +++ b/packages/db/research/agentx-stability-backfill.sql @@ -0,0 +1,347 @@ +\set ON_ERROR_STOP on + +-- Idempotently backfill AgentX stability fields from the stored request +-- timeline. The Pareto whisker uses retrospective cumulative-prefix +-- convergence; the older non-overlapping-window ranges remain available for +-- drift inspection. Neither is a confidence interval or rerun prediction. +-- +-- Required psql variables: +-- expected_branch_id Neon branch id this write is allowed to target +-- Optional psql variables: +-- min_id inclusive benchmark_results.id (default 0) +-- max_id inclusive benchmark_results.id (default bigint max) + +\if :{?expected_branch_id} +\else + \echo 'Refusing to run: expected_branch_id is required.' + \quit 3 +\endif + +\if :{?min_id} +\else + \set min_id 0 +\endif + +\if :{?max_id} +\else + \set max_id 9223372036854775807 +\endif + +SELECT current_setting('neon.branch_id', true) = :'expected_branch_id' AS on_expected_branch +\gset + +\if :on_expected_branch + \echo 'Verified Neon branch' :expected_branch_id +\else + \echo 'Refusing to run: connected Neon branch does not match' :expected_branch_id + \quit 3 +\endif + +BEGIN; +SET LOCAL lock_timeout = '5s'; +SET LOCAL statement_timeout = '15min'; + +CREATE TEMP TABLE _agentx_stability_patch ON COMMIT DROP AS +WITH candidates AS ( + SELECT + br.id, + floor((br.metrics->>'duration_seconds')::float8 / 600)::int AS expected_windows, + (floor((br.metrics->>'duration_seconds')::float8 / 300) * 300)::int + AS convergence_horizon_seconds, + atr.request_timeline + FROM benchmark_results br + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND br.id BETWEEN :min_id::bigint AND :max_id::bigint + AND (br.metrics->>'duration_seconds')::float8 >= 1200 + AND jsonb_typeof(atr.request_timeline->'requests') = 'array' +), raw AS ( + SELECT + c.id, + c.expected_windows, + COALESCE( + NULLIF(r->>'credit', '')::float8, + NULLIF(r->>'start', '')::float8 + ) AS timestamp_ns, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS root_id, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + CASE + WHEN NULLIF(r->>'end', '')::float8 > NULLIF(r->>'start', '')::float8 + THEN (NULLIF(r->>'end', '')::float8 - NULLIF(r->>'start', '')::float8) / 1e9 + END AS e2el_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS itl_s + FROM candidates c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE COALESCE(NULLIF(r->>'phase', ''), 'profiling') = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), timed AS ( + SELECT + *, + min(timestamp_ns) OVER (PARTITION BY id) AS origin_ns + FROM raw + WHERE timestamp_ns IS NOT NULL +), windowed AS ( + SELECT + *, + floor((timestamp_ns - origin_ns) / 600e9)::int AS window_index + FROM timed +), window_metrics AS ( + SELECT + id, + expected_windows, + window_index, + count(*)::int AS request_count, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s > 0) AS p75_ttft, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s > 0) AS p90_ttft, + percentile_cont(.75) WITHIN GROUP (ORDER BY e2el_s) + FILTER (WHERE e2el_s > 0) AS p75_e2el, + percentile_cont(.9) WITHIN GROUP (ORDER BY e2el_s) + FILTER (WHERE e2el_s > 0) AS p90_e2el, + percentile_cont(.75) WITHIN GROUP (ORDER BY itl_s) + FILTER (WHERE itl_s > 0) AS p75_itl, + percentile_cont(.9) WITHIN GROUP (ORDER BY itl_s) + FILTER (WHERE itl_s > 0) AS p90_itl + FROM windowed + WHERE window_index >= 0 + AND window_index < expected_windows + GROUP BY id, expected_windows, window_index +), window_summary AS ( + SELECT + id, + max(expected_windows)::int AS expected_window_count, + count(*)::int AS observed_window_count, + min(request_count)::int AS min_window_requests, + min(p75_ttft) AS p75_ttft_min, + max(p75_ttft) AS p75_ttft_max, + min(p90_ttft) AS p90_ttft_min, + max(p90_ttft) AS p90_ttft_max, + min(p75_e2el) AS p75_e2el_min, + max(p75_e2el) AS p75_e2el_max, + min(p90_e2el) AS p90_e2el_min, + max(p90_e2el) AS p90_e2el_max, + CASE WHEN max(p75_itl) > 0 THEN 1 / max(p75_itl) END AS p75_intvty_min, + CASE WHEN min(p75_itl) > 0 THEN 1 / min(p75_itl) END AS p75_intvty_max, + CASE WHEN max(p90_itl) > 0 THEN 1 / max(p90_itl) END AS p90_intvty_min, + CASE WHEN min(p90_itl) > 0 THEN 1 / min(p90_itl) END AS p90_intvty_max + FROM window_metrics + GROUP BY id +), prefix_metrics AS ( + SELECT + c.id, + c.convergence_horizon_seconds, + checkpoint.seconds::int AS checkpoint_seconds, + count(t.timestamp_ns)::int AS request_count, + count(*) FILTER (WHERE t.ttft_s > 0)::int AS ttft_request_count, + count(*) FILTER (WHERE t.e2el_s > 0)::int AS e2el_request_count, + count(*) FILTER (WHERE t.itl_s > 0)::int AS intvty_request_count, + percentile_cont(.75) WITHIN GROUP (ORDER BY t.ttft_s) + FILTER (WHERE t.ttft_s > 0) AS p75_ttft, + percentile_cont(.9) WITHIN GROUP (ORDER BY t.ttft_s) + FILTER (WHERE t.ttft_s > 0) AS p90_ttft, + percentile_cont(.75) WITHIN GROUP (ORDER BY t.e2el_s) + FILTER (WHERE t.e2el_s > 0) AS p75_e2el, + percentile_cont(.9) WITHIN GROUP (ORDER BY t.e2el_s) + FILTER (WHERE t.e2el_s > 0) AS p90_e2el, + percentile_cont(.75) WITHIN GROUP (ORDER BY t.itl_s) + FILTER (WHERE t.itl_s > 0) AS p75_itl, + percentile_cont(.9) WITHIN GROUP (ORDER BY t.itl_s) + FILTER (WHERE t.itl_s > 0) AS p90_itl + FROM candidates c + CROSS JOIN LATERAL generate_series( + 300, + c.convergence_horizon_seconds, + 300 + ) checkpoint(seconds) + JOIN timed t + ON t.id = c.id + AND t.timestamp_ns >= t.origin_ns + AND t.timestamp_ns < t.origin_ns + checkpoint.seconds * 1e9 + GROUP BY c.id, c.convergence_horizon_seconds, checkpoint.seconds +), prefix_values AS ( + SELECT + pm.id, + pm.convergence_horizon_seconds, + pm.checkpoint_seconds, + metric.request_count, + metric.metric_key, + metric.metric_value + FROM prefix_metrics pm + CROSS JOIN LATERAL (VALUES + ('p75_ttft', pm.p75_ttft, pm.ttft_request_count), + ('p90_ttft', pm.p90_ttft, pm.ttft_request_count), + ('p75_e2el', pm.p75_e2el, pm.e2el_request_count), + ('p90_e2el', pm.p90_e2el, pm.e2el_request_count), + ( + 'p75_intvty', + CASE WHEN pm.p75_itl > 0 THEN 1 / pm.p75_itl END, + pm.intvty_request_count + ), + ( + 'p90_intvty', + CASE WHEN pm.p90_itl > 0 THEN 1 / pm.p90_itl END, + pm.intvty_request_count + ) + ) metric(metric_key, metric_value, request_count) +), final_values AS ( + SELECT + id, + metric_key, + metric_value AS final_value + FROM prefix_values + WHERE checkpoint_seconds = convergence_horizon_seconds +), stabilization_candidates AS ( + SELECT + pv.id, + pv.metric_key, + pv.checkpoint_seconds, + pv.request_count + FROM prefix_values pv + JOIN final_values fv USING (id, metric_key) + WHERE pv.convergence_horizon_seconds - pv.checkpoint_seconds >= 1200 + AND pv.metric_value > 0 + AND fv.final_value > 0 + AND NOT EXISTS ( + SELECT 1 + FROM prefix_values later + WHERE later.id = pv.id + AND later.metric_key = pv.metric_key + AND later.checkpoint_seconds >= pv.checkpoint_seconds + AND ( + later.metric_value IS NULL + OR later.metric_value <= 0 + OR abs(ln(later.metric_value / fv.final_value)) + > ln(1.05::float8) + 1e-12 + ) + ) +), stabilization AS ( + SELECT DISTINCT ON (id, metric_key) + id, + metric_key, + checkpoint_seconds AS time_seconds, + request_count + FROM stabilization_candidates + ORDER BY id, metric_key, checkpoint_seconds +), convergence_summary AS ( + SELECT + s.id, + s.metric_key, + s.time_seconds, + s.request_count, + min(pv.metric_value) AS min_value, + max(pv.metric_value) AS max_value, + max(abs(pv.metric_value / fv.final_value - 1)) AS max_relative_deviation + FROM stabilization s + JOIN prefix_values pv + ON pv.id = s.id + AND pv.metric_key = s.metric_key + AND pv.checkpoint_seconds >= s.time_seconds + JOIN final_values fv + ON fv.id = s.id + AND fv.metric_key = s.metric_key + GROUP BY s.id, s.metric_key, s.time_seconds, s.request_count +), convergence_kv AS ( + SELECT + id, + 'convergence_' || metric_key || '_time_seconds' AS key, + to_jsonb(time_seconds) AS value + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_requests', + to_jsonb(request_count) + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_min', + to_jsonb(round(min_value::numeric, 5)) + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_max', + to_jsonb(round(max_value::numeric, 5)) + FROM convergence_summary + UNION ALL + SELECT + id, + 'convergence_' || metric_key || '_max_relative_deviation', + to_jsonb(round(max_relative_deviation::numeric, 5)) + FROM convergence_summary +), convergence_patches AS ( + SELECT id, jsonb_object_agg(key, value) AS patch + FROM convergence_kv + GROUP BY id +), root_counts AS ( + SELECT id, root_id, count(*)::bigint AS request_count + FROM raw + GROUP BY id, root_id +), root_summary AS ( + SELECT + id, + count(*)::int AS root_trajectory_count, + (sum(request_count)::float8 * sum(request_count)) + / NULLIF(sum(request_count::float8 * request_count), 0) + AS root_trajectory_kish_effective_count, + max(request_count)::float8 / NULLIF(sum(request_count), 0) + AS root_trajectory_largest_share + FROM root_counts + GROUP BY id +), patches AS ( + SELECT + ws.id, + jsonb_strip_nulls(jsonb_build_object( + 'observed_window_seconds', 600, + 'observed_window_expected_count', ws.expected_window_count, + 'observed_window_count', ws.observed_window_count, + 'observed_window_min_requests', ws.min_window_requests, + 'root_trajectory_count', rs.root_trajectory_count, + 'root_trajectory_kish_effective_count', + round(rs.root_trajectory_kish_effective_count::numeric, 5), + 'root_trajectory_largest_share', + round(rs.root_trajectory_largest_share::numeric, 5), + 'observed_window_p75_ttft_min', round(ws.p75_ttft_min::numeric, 5), + 'observed_window_p75_ttft_max', round(ws.p75_ttft_max::numeric, 5), + 'observed_window_p90_ttft_min', round(ws.p90_ttft_min::numeric, 5), + 'observed_window_p90_ttft_max', round(ws.p90_ttft_max::numeric, 5), + 'observed_window_p75_e2el_min', round(ws.p75_e2el_min::numeric, 5), + 'observed_window_p75_e2el_max', round(ws.p75_e2el_max::numeric, 5), + 'observed_window_p90_e2el_min', round(ws.p90_e2el_min::numeric, 5), + 'observed_window_p90_e2el_max', round(ws.p90_e2el_max::numeric, 5), + 'observed_window_p75_intvty_min', round(ws.p75_intvty_min::numeric, 5), + 'observed_window_p75_intvty_max', round(ws.p75_intvty_max::numeric, 5), + 'observed_window_p90_intvty_min', round(ws.p90_intvty_min::numeric, 5), + 'observed_window_p90_intvty_max', round(ws.p90_intvty_max::numeric, 5), + 'convergence_checkpoint_seconds', 300, + 'convergence_tolerance_ratio', 0.05, + 'convergence_min_confirmation_seconds', 1200, + 'convergence_horizon_seconds', c.convergence_horizon_seconds + )) || COALESCE(cp.patch, '{}'::jsonb) AS patch + FROM window_summary ws + JOIN root_summary rs USING (id) + JOIN candidates c USING (id) + LEFT JOIN convergence_patches cp USING (id) +) +SELECT id, patch +FROM patches; + +WITH updated AS ( + UPDATE benchmark_results br + SET metrics = br.metrics || p.patch + FROM _agentx_stability_patch p + WHERE br.id = p.id + AND br.metrics IS DISTINCT FROM br.metrics || p.patch + RETURNING br.id +) +SELECT + (SELECT count(*) FROM _agentx_stability_patch) AS computed_rows, + count(*) AS updated_rows, + min(id) AS first_updated_id, + max(id) AS last_updated_id +FROM updated; + +COMMIT; diff --git a/packages/db/research/agentx-trace-coverage.sql b/packages/db/research/agentx-trace-coverage.sql new file mode 100644 index 00000000..f6ed4f19 --- /dev/null +++ b/packages/db/research/agentx-trace-coverage.sql @@ -0,0 +1,113 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +-- Root-trajectory exposure for overlap and repeated-configuration analyses. +COPY ( +WITH canonical AS ( + SELECT + br.id, + br.workflow_run_id, + br.date, + br.conc, + br.image, + br.offload_mode, + br.config_id, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + c.disagg, + c.is_multinode, + c.prefill_tp, + c.prefill_ep, + c.prefill_dp_attention, + c.prefill_num_workers, + c.decode_tp, + c.decode_ep, + c.decode_dp_attention, + c.decode_num_workers, + c.num_prefill_gpu, + c.num_decode_gpu, + COALESCE(ds.dataset_slugs, '') AS dataset_slugs, + atr.request_timeline + FROM benchmark_results br + JOIN configs c ON c.id = br.config_id + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + LEFT JOIN LATERAL ( + SELECT string_agg(rd.dataset_slug, ',' ORDER BY rd.dataset_slug) AS dataset_slugs + FROM run_datasets rd + WHERE rd.workflow_run_id = br.workflow_run_id + ) ds ON true + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND (br.metrics->>'duration_seconds')::float8 BETWEEN 3500 AND 3700 + AND mod(br.id, 10) = :shard + AND br.id <= :max_id +), trace_counts AS ( + SELECT + c.id, + c.workflow_run_id, + c.date, + c.conc, + c.image, + c.offload_mode, + c.config_id, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + c.disagg, + c.is_multinode, + c.prefill_tp, + c.prefill_ep, + c.prefill_dp_attention, + c.prefill_num_workers, + c.decode_tp, + c.decode_ep, + c.decode_dp_attention, + c.decode_num_workers, + c.num_prefill_gpu, + c.num_decode_gpu, + c.dataset_slugs, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS source_trace_id, + count(*)::int AS n_requests, + min((r->>'credit')::float8) AS first_credit_ns, + max((r->>'credit')::float8) AS last_credit_ns + FROM canonical c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false + GROUP BY + c.id, + c.workflow_run_id, + c.date, + c.conc, + c.image, + c.offload_mode, + c.config_id, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + c.disagg, + c.is_multinode, + c.prefill_tp, + c.prefill_ep, + c.prefill_dp_attention, + c.prefill_num_workers, + c.decode_tp, + c.decode_ep, + c.decode_dp_attention, + c.decode_num_workers, + c.num_prefill_gpu, + c.num_decode_gpu, + c.dataset_slugs, + source_trace_id +) +SELECT * +FROM trace_counts +ORDER BY id, first_credit_ns, source_trace_id +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/research/agentx-uncertainty-diagnostics.sql b/packages/db/research/agentx-uncertainty-diagnostics.sql new file mode 100644 index 00000000..e96c88db --- /dev/null +++ b/packages/db/research/agentx-uncertainty-diagnostics.sql @@ -0,0 +1,228 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +COPY ( +WITH canonical AS ( + SELECT + br.id, + br.conc, + c.hardware, + c.framework, + c.model, + c.precision, + c.spec_method, + br.offload_mode, + atr.request_timeline + FROM benchmark_results br + JOIN configs c ON c.id = br.config_id + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND (br.metrics->>'duration_seconds')::float8 BETWEEN 3500 AND 3700 + -- Run with `psql -v shard=0` through `-v shard=9`. Keeping each read + -- replica snapshot short avoids cancellation during Neon recovery. + AND mod(br.id, 10) = :shard + -- Pin this to the maximum id observed before launching the ten shards so + -- a concurrent ingest cannot make the local analysis internally mixed. + AND br.id <= :max_id +), raw AS ( + SELECT + c.id, + c.conc, + c.hardware, + c.framework, + c.model, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS cluster_id, + concat_ws( + '|', + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown'), + COALESCE(NULLIF(r->>'srcOuter', ''), NULLIF(r->>'ti', ''), '0'), + COALESCE(NULLIF(r->>'srcInner', ''), '-1') + ) AS request_signature, + NULLIF(r->>'cid', '') AS cid, + NULLIF(r->>'wid', '') AS wid, + (r->>'credit')::float8 AS credit_ns, + (r->>'start')::float8 AS start_ns, + (r->>'end')::float8 AS end_ns, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS tpot_s, + NULLIF(r->>'isl', '')::float8 AS isl, + NULLIF(r->>'osl', '')::float8 AS osl + FROM canonical c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), requests AS ( + SELECT *, min(credit_ns) OVER (PARTITION BY id) AS origin_ns + FROM raw +), cluster_counts AS ( + SELECT id, cluster_id, count(*) AS n + FROM requests + GROUP BY id, cluster_id +), cluster_summary AS ( + SELECT + id, + sum(n)::int AS n_requests, + count(*)::int AS source_trajectories, + (sum(n)::float8 * sum(n)) / sum(n::float8 * n) AS kish_source_trajectories, + max(n)::float8 / sum(n) AS largest_source_share + FROM cluster_counts + GROUP BY id +), other_counts AS ( + SELECT + id, + count(DISTINCT cid)::int AS replay_trajectories, + count(DISTINCT wid)::int AS workers + FROM requests + GROUP BY id +), point_values AS ( + SELECT + id, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) AS e2e_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns < 1800e9) AS ttft_first_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns >= 1800e9) AS ttft_second_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns < 1800e9) AS e2e_first_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns >= 1800e9) AS e2e_second_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns < 1800e9) AS tpot_first_half, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns >= 1800e9) AS tpot_second_half + FROM requests + GROUP BY id +), block_values AS ( + SELECT + id, + floor((credit_ns - origin_ns) / 600e9)::int AS block, + count(*)::int AS n, + count(DISTINCT cluster_id)::int AS source_trajectories, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) AS e2e_p90, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p90 + FROM requests + WHERE floor((credit_ns - origin_ns) / 600e9) BETWEEN 0 AND 5 + GROUP BY id, block +), block_summary AS ( + SELECT + id, + count(*)::int AS n_blocks, + min(n)::int AS min_block_n, + min(source_trajectories)::int AS min_block_trajectories, + min(ttft_p75) AS ttft_p75_min, + max(ttft_p75) AS ttft_p75_max, + min(ttft_p90) AS ttft_p90_min, + max(ttft_p90) AS ttft_p90_max, + min(e2e_p90) AS e2e_p90_min, + max(e2e_p90) AS e2e_p90_max, + min(tpot_p90) AS tpot_p90_min, + max(tpot_p90) AS tpot_p90_max + FROM block_values + GROUP BY id +), signature_halves AS ( + SELECT + id, + cluster_id, + request_signature, + count(*) FILTER (WHERE credit_ns - origin_ns < 1800e9)::int AS first_half_n, + count(*) FILTER (WHERE credit_ns - origin_ns >= 1800e9)::int AS second_half_n, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns < 1800e9) AS ttft_first_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL AND credit_ns - origin_ns >= 1800e9) AS ttft_second_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns < 1800e9) AS e2e_first_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY (end_ns - start_ns) / 1e9) + FILTER (WHERE credit_ns - origin_ns >= 1800e9) AS e2e_second_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns < 1800e9) AS tpot_first_half, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0 AND credit_ns - origin_ns >= 1800e9) AS tpot_second_half + FROM requests + GROUP BY id, cluster_id, request_signature +), paired_signature_summary AS ( + SELECT + id, + count(*) FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND ttft_first_half > 0 AND ttft_second_half > 0 + )::int AS paired_signatures, + count(DISTINCT cluster_id) FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND ttft_first_half > 0 AND ttft_second_half > 0 + )::int AS paired_source_trajectories, + percentile_cont(.5) WITHIN GROUP (ORDER BY ttft_second_half / ttft_first_half) + FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND ttft_first_half > 0 AND ttft_second_half > 0 + ) AS matched_ttft_half_ratio, + percentile_cont(.5) WITHIN GROUP (ORDER BY e2e_second_half / e2e_first_half) + FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND e2e_first_half > 0 AND e2e_second_half > 0 + ) AS matched_e2e_half_ratio, + percentile_cont(.5) WITHIN GROUP (ORDER BY tpot_second_half / tpot_first_half) + FILTER ( + WHERE first_half_n > 0 AND second_half_n > 0 + AND tpot_first_half > 0 AND tpot_second_half > 0 + ) AS matched_tpot_half_ratio + FROM signature_halves + GROUP BY id +) +SELECT + c.id, + c.conc, + c.hardware, + c.framework, + c.model, + cs.n_requests, + cs.source_trajectories, + cs.kish_source_trajectories, + cs.largest_source_share, + oc.replay_trajectories, + oc.workers, + pv.ttft_p75, + pv.ttft_p90, + pv.e2e_p90, + CASE WHEN pv.tpot_p90 > 0 THEN 1 / pv.tpot_p90 END AS interactivity_p90, + bs.n_blocks, + bs.min_block_n, + bs.min_block_trajectories, + bs.ttft_p75_min, + bs.ttft_p75_max, + bs.ttft_p90_min, + bs.ttft_p90_max, + bs.e2e_p90_min, + bs.e2e_p90_max, + CASE WHEN bs.tpot_p90_max > 0 THEN 1 / bs.tpot_p90_max END AS interactivity_p90_min, + CASE WHEN bs.tpot_p90_min > 0 THEN 1 / bs.tpot_p90_min END AS interactivity_p90_max, + bs.ttft_p90_max / NULLIF(bs.ttft_p90_min, 0) AS ttft_block_ratio, + bs.e2e_p90_max / NULLIF(bs.e2e_p90_min, 0) AS e2e_block_ratio, + bs.tpot_p90_max / NULLIF(bs.tpot_p90_min, 0) AS tpot_block_ratio, + pv.ttft_second_half / NULLIF(pv.ttft_first_half, 0) AS ttft_half_ratio, + pv.e2e_second_half / NULLIF(pv.e2e_first_half, 0) AS e2e_half_ratio, + pv.tpot_second_half / NULLIF(pv.tpot_first_half, 0) AS tpot_half_ratio + ,ps.paired_signatures + ,ps.paired_source_trajectories + ,ps.matched_ttft_half_ratio + ,ps.matched_e2e_half_ratio + ,ps.matched_tpot_half_ratio +FROM canonical c +JOIN cluster_summary cs USING (id) +JOIN other_counts oc USING (id) +JOIN point_values pv USING (id) +JOIN block_summary bs USING (id) +JOIN paired_signature_summary ps USING (id) +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/research/agentx-window-sensitivity.sql b/packages/db/research/agentx-window-sensitivity.sql new file mode 100644 index 00000000..c5383859 --- /dev/null +++ b/packages/db/research/agentx-window-sensitivity.sql @@ -0,0 +1,127 @@ +\set ON_ERROR_STOP on +SET statement_timeout = '600s'; + +-- Non-overlapping, equal-duration slices for the pinned one-hour AgentX cohort. +-- Run with psql variables `shard=0..9` and `max_id=`. +-- Equal-duration slices avoid treating a short final wall-clock bucket as if it +-- had the same exposure as a complete bucket. +COPY ( +WITH canonical AS ( + SELECT + br.id, + br.conc, + c.hardware, + c.framework, + c.model, + atr.request_timeline + FROM benchmark_results br + JOIN configs c ON c.id = br.config_id + JOIN agentic_trace_replay atr ON atr.id = br.trace_replay_id + WHERE br.benchmark_type = 'agentic_traces' + AND br.error IS NULL + AND (br.metrics->>'duration_seconds')::float8 BETWEEN 3500 AND 3700 + AND mod(br.id, 10) = :shard + AND br.id <= :max_id +), raw AS ( + SELECT + c.id, + c.conc, + c.hardware, + c.framework, + c.model, + COALESCE(NULLIF(r->>'srcTrace', ''), NULLIF(r->>'cid', ''), 'unknown') AS cluster_id, + (r->>'credit')::float8 AS credit_ns, + (r->>'end')::float8 AS end_ns, + NULLIF(r->>'ttftMs', '')::float8 / 1000 AS ttft_s, + ((r->>'end')::float8 - (r->>'start')::float8) / 1e9 AS e2e_s, + NULLIF(r->>'tpotMs', '')::float8 / 1000 AS tpot_s, + NULLIF(r->>'isl', '')::float8 AS isl, + NULLIF(r->>'osl', '')::float8 AS osl + FROM canonical c + CROSS JOIN LATERAL jsonb_array_elements(c.request_timeline->'requests') r + WHERE r->>'phase' = 'profiling' + AND COALESCE((r->>'cancelled')::boolean, false) = false +), timed AS ( + SELECT + *, + min(credit_ns) OVER (PARTITION BY id) AS origin_ns, + max(credit_ns) OVER (PARTITION BY id) - min(credit_ns) OVER (PARTITION BY id) AS span_ns + FROM raw +), expanded AS ( + SELECT + t.*, + w.window_minutes, + w.n_blocks, + least( + w.n_blocks - 1, + floor((t.credit_ns - t.origin_ns) / NULLIF(t.span_ns, 0) * w.n_blocks)::int + ) AS block + FROM timed t + CROSS JOIN ( + VALUES + (5, 12), + (10, 6), + (15, 4), + (20, 3) + ) AS w(window_minutes, n_blocks) +), cluster_counts AS ( + SELECT id, window_minutes, block, cluster_id, count(*) AS n + FROM expanded + GROUP BY id, window_minutes, block, cluster_id +), cluster_summary AS ( + SELECT + id, + window_minutes, + block, + count(*)::int AS source_trajectories, + (sum(n)::float8 * sum(n)) / NULLIF(sum(n::float8 * n), 0) AS kish_source_trajectories, + max(n)::float8 / NULLIF(sum(n), 0) AS largest_source_share + FROM cluster_counts + GROUP BY id, window_minutes, block +), block_values AS ( + SELECT + id, + conc, + hardware, + framework, + model, + window_minutes, + n_blocks, + block, + count(*)::int AS n_requests, + percentile_cont(.75) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY ttft_s) + FILTER (WHERE ttft_s IS NOT NULL) AS ttft_p90, + percentile_cont(.75) WITHIN GROUP (ORDER BY e2e_s) AS e2e_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY e2e_s) AS e2e_p90, + percentile_cont(.75) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p75, + percentile_cont(.9) WITHIN GROUP (ORDER BY tpot_s) + FILTER (WHERE tpot_s > 0) AS tpot_p90, + avg(isl) AS isl_mean, + percentile_cont(.5) WITHIN GROUP (ORDER BY isl) + FILTER (WHERE isl IS NOT NULL) AS isl_p50, + percentile_cont(.9) WITHIN GROUP (ORDER BY isl) + FILTER (WHERE isl IS NOT NULL) AS isl_p90, + avg(osl) AS osl_mean, + percentile_cont(.5) WITHIN GROUP (ORDER BY osl) + FILTER (WHERE osl IS NOT NULL) AS osl_p50, + percentile_cont(.9) WITHIN GROUP (ORDER BY osl) + FILTER (WHERE osl IS NOT NULL) AS osl_p90, + sum(COALESCE(isl, 0) + COALESCE(osl, 0)) / (window_minutes * 60) AS token_rate + FROM expanded + WHERE block BETWEEN 0 AND n_blocks - 1 + GROUP BY id, conc, hardware, framework, model, window_minutes, n_blocks, block +) +SELECT + b.*, + c.source_trajectories, + c.kish_source_trajectories, + c.largest_source_share, + CASE WHEN b.tpot_p75 > 0 THEN 1 / b.tpot_p75 END AS interactivity_p75, + CASE WHEN b.tpot_p90 > 0 THEN 1 / b.tpot_p90 END AS interactivity_p90 +FROM block_values b +JOIN cluster_summary c USING (id, window_minutes, block) +ORDER BY id, window_minutes, block +) TO STDOUT WITH (FORMAT csv, HEADER true); diff --git a/packages/db/src/etl/agentic-v3-flatten.ts b/packages/db/src/etl/agentic-v3-flatten.ts index a3c223af..c9e3b6b1 100644 --- a/packages/db/src/etl/agentic-v3-flatten.ts +++ b/packages/db/src/etl/agentic-v3-flatten.ts @@ -63,6 +63,37 @@ const V3_SCALAR_PATHS: [string[], string][] = [ [['server_metrics', 'tokens', 'prompt_total'], 'total_prompt_tokens'], [['server_metrics', 'tokens', 'generation_total'], 'total_generation_tokens'], [['server_metrics', 'tokens', 'requests_completed'], 'total_requests_completed'], + // One-run AgentX coverage and non-overlapping-window diagnostics. These + // describe the realized hour; they are not confidence intervals. + [['request_metrics', 'stability', 'window_seconds'], 'observed_window_seconds'], + [['request_metrics', 'stability', 'expected_window_count'], 'observed_window_expected_count'], + [['request_metrics', 'stability', 'observed_window_count'], 'observed_window_count'], + [['request_metrics', 'stability', 'min_window_requests'], 'observed_window_min_requests'], + [['request_metrics', 'stability', 'root_trajectory_count'], 'root_trajectory_count'], + [ + ['request_metrics', 'stability', 'root_trajectory_kish_effective_count'], + 'root_trajectory_kish_effective_count', + ], + [ + ['request_metrics', 'stability', 'root_trajectory_largest_share'], + 'root_trajectory_largest_share', + ], + [ + ['request_metrics', 'stability', 'convergence', 'checkpoint_seconds'], + 'convergence_checkpoint_seconds', + ], + [ + ['request_metrics', 'stability', 'convergence', 'tolerance_ratio'], + 'convergence_tolerance_ratio', + ], + [ + ['request_metrics', 'stability', 'convergence', 'min_confirmation_seconds'], + 'convergence_min_confirmation_seconds', + ], + [ + ['request_metrics', 'stability', 'convergence', 'horizon_seconds'], + 'convergence_horizon_seconds', + ], // Deliberately NOT mapped (yet): cache.overall/prefix_cache_hits/queries, // kv_cache.cpu_*, tokens.prompt_by_source, sources[] — new v3 detail we don't // consume anywhere; add here + METRIC_KEYS when a view needs them. @@ -127,5 +158,46 @@ export function flattenAgenticAggRow(row: Record): Record = {}): Record { expect(m.total_prompt_tokens).toBe(261750519); expect(m.total_generation_tokens).toBe(1422696); expect(m.total_requests_completed).toBe(1648); + // realized coverage + non-overlapping-window diagnostics + expect(m.observed_window_seconds).toBe(600); + expect(m.observed_window_count).toBe(6); + expect(m.observed_window_min_requests).toBe(190); + expect(m.root_trajectory_count).toBe(25); + expect(m.root_trajectory_kish_effective_count).toBeCloseTo(14.767, 3); + expect(m.observed_window_p90_ttft_min).toBeCloseTo(1.416, 3); + expect(m.observed_window_p90_ttft_max).toBeCloseTo(4.114, 3); + expect(m.observed_window_p75_e2el_max).toBeCloseTo(79.2, 3); + expect(m.observed_window_p90_intvty_min).toBeCloseTo(22.621, 3); + // retrospective cumulative-prefix convergence + expect(m.convergence_checkpoint_seconds).toBe(300); + expect(m.convergence_tolerance_ratio).toBe(0.05); + expect(m.convergence_min_confirmation_seconds).toBe(1200); + expect(m.convergence_horizon_seconds).toBe(3600); + expect(m.convergence_p75_ttft_time_seconds).toBe(900); + expect(m.convergence_p75_ttft_requests).toBe(300); + expect(m.convergence_p75_ttft_min).toBeCloseTo(1.21, 5); + expect(m.convergence_p75_ttft_max_relative_deviation).toBeCloseTo(0.031, 5); + expect(m.convergence_p90_intvty_time_seconds).toBe(1200); + expect(m.convergence_p90_intvty_requests).toBe(407); + expect(m.convergence_p90_intvty_min).toBeCloseTo(43.36036, 5); + expect(m.convergence_p90_intvty_max).toBeCloseTo(46.97824, 5); + expect(m.convergence_p90_intvty_max_relative_deviation).toBeCloseTo(0.04385, 5); + expect(m).not.toHaveProperty('convergence_p90_ttft_time_seconds'); // nested containers must not leak into metrics expect(m).not.toHaveProperty('request_metrics'); expect(m).not.toHaveProperty('server_metrics');