Skip to content

Commit fb033aa

Browse files
edwingao28edwingao28
committed
feat(overview): aggregate executive inference metrics
中文：为高管总览页聚合推理指标。
1 parent bd4ca47 commit fb033aa

2 files changed

Lines changed: 365 additions & 0 deletions

File tree

‎packages/app/src/lib/overview-data.test.ts‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
import { describe, expect, it } from 'vitest';
2+
3+
import type { BenchmarkRow } from './api';
4+
import { Model, Precision } from './data-mappings';
5+
import {
6+
buildOverviewModelSummary,
7+
OVERVIEW_TIERS,
8+
selectOverviewPrecision,
9+
} from './overview-data';
10+
11+
let nextId = 1;
12+
13+
function row(overrides: Partial<BenchmarkRow> = {}): BenchmarkRow {
14+
return {
15+
id: nextId++,
16+
hardware: 'b200',
17+
framework: 'sglang',
18+
model: 'qwen3.5',
19+
precision: 'fp8',
20+
spec_method: 'mtp',
21+
disagg: false,
22+
is_multinode: false,
23+
prefill_tp: 8,
24+
prefill_ep: 1,
25+
prefill_dp_attention: false,
26+
prefill_num_workers: 1,
27+
decode_tp: 8,
28+
decode_ep: 1,
29+
decode_dp_attention: false,
30+
decode_num_workers: 1,
31+
num_prefill_gpu: 8,
32+
num_decode_gpu: 8,
33+
benchmark_type: 'single_turn',
34+
isl: 8192,
35+
osl: 1024,
36+
conc: 16,
37+
offload_mode: 'off',
38+
image: null,
39+
metrics: { median_intvty: 50, output_tput_per_gpu: 1000 },
40+
date: '2026-07-20',
41+
run_url: null,
42+
...overrides,
43+
};
44+
}
45+
46+
describe('selectOverviewPrecision', () => {
47+
it('chooses the precision with the most distinct curves and uses registry order for ties', () => {
48+
const rows = [
49+
row({ hardware: 'b200', precision: Precision.FP4 }),
50+
row({ hardware: 'b200', precision: Precision.FP8 }),
51+
row({ hardware: 'gb200', precision: Precision.FP8 }),
52+
row({ hardware: 'b200', precision: Precision.FP4, conc: 32 }),
53+
];
54+
55+
expect(selectOverviewPrecision(rows)).toBe(Precision.FP8);
56+
expect(selectOverviewPrecision(rows.slice(0, 2))).toBe(Precision.FP4);
57+
});
58+
});
59+
60+
describe('buildOverviewModelSummary', () => {
61+
it('selects speculative rows independently for each hardware', () => {
62+
const summary = buildOverviewModelSummary(Model.Qwen3_5, [
63+
row({
64+
hardware: 'b200',
65+
spec_method: 'none',
66+
metrics: { median_intvty: 50, output_tput_per_gpu: 800 },
67+
}),
68+
row({
69+
hardware: 'b200',
70+
spec_method: 'mtp',
71+
metrics: { median_intvty: 50, output_tput_per_gpu: 1000 },
72+
}),
73+
row({
74+
hardware: 'mi355x',
75+
spec_method: 'none',
76+
metrics: { median_intvty: 50, output_tput_per_gpu: 900 },
77+
}),
78+
]);
79+
80+
const b200 = summary.hardwareResults.find((result) => result.hardware === 'b200');
81+
const mi355x = summary.hardwareResults.find((result) => result.hardware === 'mi355x');
82+
83+
expect(b200).toMatchObject({
84+
hardwareLabel: 'B200',
85+
decodeMode: 'speculative',
86+
decodeLabel: 'MTP',
87+
});
88+
expect(b200?.tierValues.find(({ tier }) => tier === 50)?.value).toBe(1000);
89+
expect(mi355x).toMatchObject({
90+
hardwareLabel: 'MI355X',
91+
decodeMode: 'standard_fallback',
92+
decodeLabel: 'Standard decode fallback',
93+
});
94+
expect(mi355x?.tierValues.find(({ tier }) => tier === 50)?.value).toBe(900);
95+
});
96+
97+
it('filters to single-turn 8K/1K data and ranks hardware with the PR #601 score', () => {
98+
const b200Throughput = [1200, 1000, 800, 600];
99+
const gb200Throughput = [1000, 800, 600, 400];
100+
const rows = [
101+
...OVERVIEW_TIERS.map((tier, index) =>
102+
row({
103+
hardware: 'b200',
104+
conc: index + 1,
105+
metrics: { median_intvty: tier, output_tput_per_gpu: b200Throughput[index] },
106+
}),
107+
),
108+
...OVERVIEW_TIERS.map((tier, index) =>
109+
row({
110+
hardware: 'gb200',
111+
conc: index + 1,
112+
metrics: { median_intvty: tier, output_tput_per_gpu: gb200Throughput[index] },
113+
}),
114+
),
115+
row({
116+
hardware: 'mi355x',
117+
isl: 1024,
118+
osl: 1024,
119+
metrics: { median_intvty: 50, output_tput_per_gpu: 10_000 },
120+
}),
121+
];
122+
123+
const summary = buildOverviewModelSummary(Model.Qwen3_5, rows);
124+
125+
expect(summary.hardwareResults.map(({ hardware }) => hardware)).toEqual(['b200', 'gb200']);
126+
expect(summary.winner).toMatchObject({ hardware: 'b200', score: 1010 });
127+
expect(summary.runnerUp).toMatchObject({ hardware: 'gb200', score: 810 });
128+
expect(summary.winner?.tierValues.map(({ tier }) => tier)).toEqual([30, 50, 75, 100]);
129+
expect(summary.runnerUpGapPercent).toBeCloseTo((1010 / 810 - 1) * 100);
130+
expect(summary.runnerUpGapPercent).toBeGreaterThan(0);
131+
});
132+
133+
it('keeps an active model visible when no 8K/1K single-turn data exists', () => {
134+
expect(buildOverviewModelSummary(Model.GLM_5_2, [])).toEqual({
135+
model: Model.GLM_5_2,
136+
modelLabel: 'GLM5.2',
137+
selectedPrecision: null,
138+
winner: null,
139+
runnerUp: null,
140+
runnerUpGapPercent: null,
141+
hardwareResults: [],
142+
emptyReason: 'no_8k1k_single_turn_data',
143+
});
144+
});
145+
146+
it('turns unreachable zeroes into null while retaining low-boundary clamps', () => {
147+
const summary = buildOverviewModelSummary(Model.Qwen3_5, [
148+
row({ metrics: { median_intvty: 40, output_tput_per_gpu: 800 } }),
149+
row({ metrics: { median_intvty: 80, output_tput_per_gpu: 200 } }),
150+
]);
151+
152+
const low = summary.winner?.tierValues.find(({ tier }) => tier === 30);
153+
const high = summary.winner?.tierValues.find(({ tier }) => tier === 100);
154+
155+
expect(low).toEqual({ tier: 30, value: 800, boundary: 'clamped_low' });
156+
expect(high).toEqual({ tier: 100, value: null, boundary: 'unreachable' });
157+
});
158+
159+
it('uses the registry label for MiniMax M3 speculative decoding', () => {
160+
const summary = buildOverviewModelSummary(Model.MiniMax_M3, [
161+
row({ model: 'minimaxm3', spec_method: 'mtp' }),
162+
]);
163+
164+
expect(summary.winner?.decodeLabel).toBe('M3 EAGLE');
165+
});
166+
});

‎packages/app/src/lib/overview-data.ts‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
import { resolveFrameworkPartLabel } from '@semianalysisai/inferencex-constants';
2+
3+
import type { BenchmarkRow } from './api';
4+
import { dedupeRowsToLatestPerConfig } from './benchmark-series';
5+
import { getHardwareConfig, getModelSortIndex } from './constants';
6+
import { getModelLabel, PRECISION_OPTIONS, type Model, type Precision } from './data-mappings';
7+
import {
8+
computeTcoFeed,
9+
computeTcoScores,
10+
DEFAULT_TIER_WEIGHTS,
11+
type TcoTierBoundary,
12+
} from './tco-feed';
13+
14+
export const OVERVIEW_WORKLOAD = { isl: 8192, osl: 1024 } as const;
15+
export const OVERVIEW_TIERS = [30, 50, 75, 100] as const;
16+
17+
export interface OverviewTierValue {
18+
tier: number;
19+
value: number | null;
20+
boundary: TcoTierBoundary;
21+
}
22+
23+
export interface OverviewHardwareResult {
24+
hardware: string;
25+
hardwareLabel: string;
26+
precision: string;
27+
decodeMode: 'speculative' | 'standard_fallback';
28+
decodeLabel: string;
29+
score: number;
30+
tierValues: OverviewTierValue[];
31+
latestDate: string;
32+
oldestFrontierDate: string;
33+
}
34+
35+
export interface OverviewModelSummary {
36+
model: Model;
37+
modelLabel: string;
38+
selectedPrecision: string | null;
39+
winner: OverviewHardwareResult | null;
40+
runnerUp: OverviewHardwareResult | null;
41+
runnerUpGapPercent: number | null;
42+
hardwareResults: OverviewHardwareResult[];
43+
emptyReason: 'no_8k1k_single_turn_data' | null;
44+
}
45+
46+
export interface OverviewPageData {
47+
models: OverviewModelSummary[];
48+
latestDate: string | null;
49+
}
50+
51+
function precisionRank(precision: string): number {
52+
const index = PRECISION_OPTIONS.indexOf(precision as Precision);
53+
return index === -1 ? PRECISION_OPTIONS.length : index;
54+
}
55+
56+
export function selectOverviewPrecision(rows: BenchmarkRow[]): string | null {
57+
const curvesByPrecision = new Map<string, Set<string>>();
58+
59+
for (const row of rows) {
60+
let curves = curvesByPrecision.get(row.precision);
61+
if (!curves) {
62+
curves = new Set<string>();
63+
curvesByPrecision.set(row.precision, curves);
64+
}
65+
curves.add(
66+
`${row.hardware}|${row.framework}|${row.spec_method}|${row.disagg}|${row.offload_mode}`,
67+
);
68+
}
69+
70+
return (
71+
[...curvesByPrecision.entries()]
72+
.toSorted(
73+
([precisionA, curvesA], [precisionB, curvesB]) =>
74+
curvesB.size - curvesA.size ||
75+
precisionRank(precisionA) - precisionRank(precisionB) ||
76+
precisionA.localeCompare(precisionB),
77+
)
78+
.at(0)?.[0] ?? null
79+
);
80+
}
81+
82+
type DecodePolicy = Pick<OverviewHardwareResult, 'decodeMode' | 'decodeLabel'>;
83+
84+
function emptyModelSummary(model: Model, selectedPrecision: string | null): OverviewModelSummary {
85+
return {
86+
model,
87+
modelLabel: getModelLabel(model),
88+
selectedPrecision,
89+
winner: null,
90+
runnerUp: null,
91+
runnerUpGapPercent: null,
92+
hardwareResults: [],
93+
emptyReason: 'no_8k1k_single_turn_data',
94+
};
95+
}
96+
97+
export function buildOverviewModelSummary(
98+
model: Model,
99+
rows: BenchmarkRow[],
100+
): OverviewModelSummary {
101+
const workloadRows = rows.filter(
102+
(row) =>
103+
row.benchmark_type === 'single_turn' &&
104+
row.isl === OVERVIEW_WORKLOAD.isl &&
105+
row.osl === OVERVIEW_WORKLOAD.osl,
106+
);
107+
const dedupedRows = dedupeRowsToLatestPerConfig(workloadRows);
108+
const selectedPrecision = selectOverviewPrecision(dedupedRows);
109+
if (selectedPrecision === null) return emptyModelSummary(model, null);
110+
111+
const rowsByHardware = new Map<string, BenchmarkRow[]>();
112+
for (const row of dedupedRows) {
113+
if (row.precision !== selectedPrecision) continue;
114+
const hardwareRows = rowsByHardware.get(row.hardware);
115+
if (hardwareRows) hardwareRows.push(row);
116+
else rowsByHardware.set(row.hardware, [row]);
117+
}
118+
119+
const retainedRows: BenchmarkRow[] = [];
120+
const decodePolicyByHardware = new Map<string, DecodePolicy>();
121+
for (const [hardware, hardwareRows] of rowsByHardware) {
122+
const speculativeRows = hardwareRows.filter((row) => row.spec_method !== 'none');
123+
if (speculativeRows.length > 0) {
124+
retainedRows.push(...speculativeRows);
125+
const decodeLabel = [...new Set(speculativeRows.map((row) => row.spec_method))]
126+
.toSorted((a, b) => a.localeCompare(b))
127+
.map((method) => resolveFrameworkPartLabel(model, method))
128+
.join(', ');
129+
decodePolicyByHardware.set(hardware, { decodeMode: 'speculative', decodeLabel });
130+
} else {
131+
retainedRows.push(...hardwareRows);
132+
decodePolicyByHardware.set(hardware, {
133+
decodeMode: 'standard_fallback',
134+
decodeLabel: 'Standard decode fallback',
135+
});
136+
}
137+
}
138+
139+
const feed = computeTcoFeed(retainedRows, [OVERVIEW_WORKLOAD], OVERVIEW_TIERS);
140+
const scores = computeTcoScores(
141+
feed,
142+
[OVERVIEW_WORKLOAD],
143+
OVERVIEW_TIERS,
144+
DEFAULT_TIER_WEIGHTS,
145+
[1],
146+
0,
147+
);
148+
149+
const hardwareResults = scores
150+
.map((score): OverviewHardwareResult => {
151+
const policy = decodePolicyByHardware.get(score.hardware)!;
152+
const tierValues = feed
153+
.filter((row) => row.hardware === score.hardware)
154+
.map(
155+
(row): OverviewTierValue => ({
156+
tier: row.tier,
157+
value:
158+
row.boundary === 'unreachable' && row.output_tput_per_gpu === 0
159+
? null
160+
: row.output_tput_per_gpu,
161+
boundary: row.boundary,
162+
}),
163+
);
164+
165+
return {
166+
hardware: score.hardware,
167+
hardwareLabel: getHardwareConfig(score.hardware, model).label,
168+
precision: selectedPrecision,
169+
...policy,
170+
score: score.score,
171+
tierValues,
172+
latestDate: score.latest_date,
173+
oldestFrontierDate: score.oldest_frontier_date,
174+
};
175+
})
176+
.toSorted(
177+
(a, b) =>
178+
b.score - a.score ||
179+
getModelSortIndex(a.hardware) - getModelSortIndex(b.hardware) ||
180+
a.hardware.localeCompare(b.hardware),
181+
);
182+
183+
if (hardwareResults.length === 0) return emptyModelSummary(model, selectedPrecision);
184+
185+
const winner = hardwareResults[0];
186+
const runnerUp = hardwareResults[1] ?? null;
187+
188+
return {
189+
model,
190+
modelLabel: getModelLabel(model),
191+
selectedPrecision,
192+
winner,
193+
runnerUp,
194+
runnerUpGapPercent:
195+
runnerUp && runnerUp.score > 0 ? (winner.score / runnerUp.score - 1) * 100 : null,
196+
hardwareResults,
197+
emptyReason: null,
198+
};
199+
}

0 commit comments

Comments
 (0)