From 3404379a62711d102f448052a1cbb98d5ea95fbc Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 13:43:11 -0700 Subject: [PATCH 01/13] feat(overview): add executive overview with precision-aware rankings and result-level evidence --- .prettierignore | 3 + packages/app/cypress/component/header.cy.tsx | 101 ++ packages/app/cypress/e2e/overview.cy.ts | 151 +++ .../cypress/fixtures/api/overview-rows.json | 26 + packages/app/src/app/overview/page.tsx | 40 + packages/app/src/app/sitemap.ts | 5 + packages/app/src/app/zh/overview/page.tsx | 41 + packages/app/src/components/header/header.tsx | 28 +- .../overview/overview-detail-link.tsx | 86 ++ .../src/components/overview/overview-page.tsx | 59 ++ .../overview/overview-scorecard.tsx | 736 +++++++++++++ .../app/src/components/ui/mode-toggle.tsx | 2 +- packages/app/src/hooks/useTogglableSet.ts | 29 +- packages/app/src/lib/benchmark-data.server.ts | 21 + packages/app/src/lib/compare-ssr.ts | 27 +- packages/app/src/lib/exclusion.ts | 2 +- packages/app/src/lib/i18n.test.ts | 3 + packages/app/src/lib/i18n.ts | 1 + .../src/lib/overview-config-identity.test.ts | 97 ++ .../app/src/lib/overview-config-identity.ts | 33 + packages/app/src/lib/overview-data.server.ts | 29 + packages/app/src/lib/overview-data.test.ts | 740 +++++++++++++ packages/app/src/lib/overview-data.ts | 969 ++++++++++++++++++ packages/app/src/lib/overview-links.test.ts | 100 ++ packages/app/src/lib/overview-links.ts | 89 ++ packages/app/src/lib/tco-feed.ts | 36 + packages/app/src/lib/toggle-set.ts | 32 + 27 files changed, 3427 insertions(+), 59 deletions(-) create mode 100644 .prettierignore create mode 100644 packages/app/cypress/e2e/overview.cy.ts create mode 100644 packages/app/cypress/fixtures/api/overview-rows.json create mode 100644 packages/app/src/app/overview/page.tsx create mode 100644 packages/app/src/app/zh/overview/page.tsx create mode 100644 packages/app/src/components/overview/overview-detail-link.tsx create mode 100644 packages/app/src/components/overview/overview-page.tsx create mode 100644 packages/app/src/components/overview/overview-scorecard.tsx create mode 100644 packages/app/src/lib/benchmark-data.server.ts create mode 100644 packages/app/src/lib/overview-config-identity.test.ts create mode 100644 packages/app/src/lib/overview-config-identity.ts create mode 100644 packages/app/src/lib/overview-data.server.ts create mode 100644 packages/app/src/lib/overview-data.test.ts create mode 100644 packages/app/src/lib/overview-data.ts create mode 100644 packages/app/src/lib/overview-links.test.ts create mode 100644 packages/app/src/lib/overview-links.ts create mode 100644 packages/app/src/lib/toggle-set.ts diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..3c258190 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +# Deliberately compact test fixture (one benchmark row per line, <=60 lines); +# oxfmt/prettier would pretty-print it to ~530 lines. See overview-data.test.ts drift guard. +packages/app/cypress/fixtures/api/overview-rows.json diff --git a/packages/app/cypress/component/header.cy.tsx b/packages/app/cypress/component/header.cy.tsx index 747396ea..308b31d4 100644 --- a/packages/app/cypress/component/header.cy.tsx +++ b/packages/app/cypress/component/header.cy.tsx @@ -5,10 +5,24 @@ import { PathnameContext } from 'next/dist/shared/lib/hooks-client-context.share import { Header } from '@/components/header/header'; import { ThemeProvider } from '@/components/ui/theme-provider'; +// Component specs mount outside the Next app shell, so the global stylesheet is +// never injected. The layout assertions below need real Tailwind output, and +// next-style-loader inserts before this anchor node, so both must exist before +// the stylesheet module evaluates. +const cssAnchor = document.createElement('noscript'); +cssAnchor.id = '__next_css__DO_NOT_USE__'; +document.head.append(cssAnchor); +require('@/app/globals.css'); + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, }); +/** Minimum touch target, per WCAG 2.5.8 and the `size-11` / `min-h-11` utilities. */ +const MIN_TOUCH_PX = 44; +/** Tolerance for sub-pixel layout rounding. */ +const EPSILON = 0.5; + function createMockRouter() { return { push: cy.stub(), @@ -20,6 +34,10 @@ function createMockRouter() { }; } +function rectOf(selector: string) { + return cy.get(selector).then(($el) => $el[0].getBoundingClientRect()); +} + describe('Header', () => { beforeEach(() => { const mockRouter = createMockRouter(); @@ -84,4 +102,87 @@ describe('Header', () => { cy.contains('a', 'Supporters').should('be.visible').and('have.attr', 'href', '/quotes'); }); }); + + describe('at 320x700', () => { + beforeEach(() => { + cy.viewport(320, 700); + }); + + it('hides the GitHub star control', () => { + cy.get('[data-testid="header-star-button"]').should('not.be.visible'); + }); + + it('keeps the remaining controls inside the header bounds', () => { + cy.get('[data-testid="header"]').then(($header) => { + const bounds = $header[0].getBoundingClientRect(); + const selectors = [ + '[data-testid="header-brand"]', + '[data-testid="language-toggle"]', + '[data-testid="theme-toggle"]', + '[data-testid="mobile-menu-toggle"]', + ]; + selectors.forEach((selector) => { + rectOf(selector).then((rect) => { + expect(rect.left, `${selector} left edge`).to.be.at.least(bounds.left - EPSILON); + expect(rect.right, `${selector} right edge`).to.be.at.most(bounds.right + EPSILON); + }); + }); + }); + }); + + it('gives the brand and language controls a 44px touch height', () => { + ['[data-testid="header-brand"]', '[data-testid="language-toggle"]'].forEach((selector) => { + rectOf(selector).then((rect) => { + expect(rect.height, `${selector} height`).to.be.at.least(MIN_TOUCH_PX - EPSILON); + }); + }); + }); + + it('gives the icon controls a 44px touch target in both dimensions', () => { + ['[data-testid="theme-toggle"]', '[data-testid="mobile-menu-toggle"]'].forEach((selector) => { + rectOf(selector).then((rect) => { + expect(rect.width, `${selector} width`).to.be.at.least(MIN_TOUCH_PX - EPSILON); + expect(rect.height, `${selector} height`).to.be.at.least(MIN_TOUCH_PX - EPSILON); + }); + }); + }); + + it('does not overflow horizontally', () => { + cy.get('[data-testid="header"]').then(($header) => { + const header = $header[0]; + expect(header.scrollWidth, 'header scrollWidth').to.be.at.most(header.clientWidth); + }); + }); + + it('still opens the menu and exposes its links', () => { + cy.get('[data-testid="mobile-menu-toggle"]').click(); + cy.get('[data-testid="mobile-menu"]').should('be.visible'); + cy.get('[data-testid="mobile-menu"]').within(() => { + ['Home', 'Dashboard', 'Comparisons', 'Supporters', 'Datasets', 'Articles', 'About'].forEach( + (label) => { + cy.contains('a', label).should('be.visible'); + }, + ); + }); + cy.get('[data-testid="mobile-menu"] a').each(($link) => { + const rect = $link[0].getBoundingClientRect(); + expect(rect.height, `${$link.text()} link height`).to.be.at.least(MIN_TOUCH_PX - EPSILON); + }); + }); + + it('exposes the minecraft audio toggles in the mobile menu without overflowing', () => { + cy.get('[data-testid="theme-toggle"]').click(); + cy.get('[data-testid="theme-toggle"]').click(); + cy.get('html').should('have.class', 'minecraft'); + cy.get('[data-testid="mobile-menu-toggle"]').click(); + cy.get('[data-testid="mobile-menu"]').within(() => { + cy.get('button[aria-label="Mute music"]').should('be.visible'); + cy.get('button[aria-label="Mute click sounds"]').should('be.visible'); + }); + cy.get('[data-testid="header"]').then(($header) => { + const header = $header[0]; + expect(header.scrollWidth, 'header scrollWidth').to.be.at.most(header.clientWidth); + }); + }); + }); }); diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts new file mode 100644 index 00000000..bdd92f69 --- /dev/null +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -0,0 +1,151 @@ +// Focused smoke coverage for /overview. In fixtures mode the page is served by +// the synthetic cypress/fixtures/api/overview-rows.json through the real data +// builder, so every expected value below was derived by running the assembler +// over that fixture (the drift guard in overview-data.test.ts locks the same +// values) — never eyeballed. The fixture exercises every 8.7 state: coverage-driven +// primary precision, ranked vs coverage secondary, all not-ranked reasons, and — +// uniquely — a cross-day evidence range the live dataset does not produce. + +const MODEL_LABELS = [ + 'DeepSeek V4 Pro 1.6T', + 'Kimi K2.5/2.6/2.7-Code 1T', + 'MiniMax M3 428B', + 'GLM5.2', + 'Qwen3.5 397B', +]; + +/** The page must never scroll sideways: the whole comparison has to fit. */ +function expectNoHorizontalOverflow() { + cy.document().then((doc) => { + expect(doc.documentElement.scrollWidth).to.be.lte(doc.documentElement.clientWidth); + }); +} + +/** + * A narrow document is not enough: a table parked in an `overflow-x` region + * keeps the document narrow while still scrolling sideways, so nothing inside + * the surface may be wider than its own box either. `sr-only` clips (1px) and + * non-replaced inline boxes are exempt — CSSOM reports a spurious `scrollWidth` + * for inline boxes in Firefox, and `overflow` cannot apply to them anyway. + */ +function expectNoHorizontalScroller(testId: string) { + cy.get(`[data-testid="${testId}"]`).then(([surface]) => { + const scrollers = [surface, ...surface.querySelectorAll('*')] + .filter( + (el) => + !el.classList.contains('sr-only') && + getComputedStyle(el).display !== 'inline' && + el.scrollWidth > el.clientWidth + 1, + ) + .map((el) => `${el.tagName} ${el.scrollWidth}>${el.clientWidth}`); + expect(scrollers, `horizontally scrollable inside ${testId}`).to.deep.equal([]); + }); +} + +function desktopModel(model: string) { + return cy.get(`[data-testid="overview-desktop-model"][data-model="${model}"]`); +} + +describe('Overview page', () => { + it('summarizes every active model without interactive widgets', () => { + cy.viewport(1280, 800); + cy.visit('/overview'); + + cy.contains('h1', 'AI Inference Overview').should('exist'); + cy.get('[data-testid="overview-desktop-matrix"]').should('be.visible'); + cy.get('[data-testid="overview-desktop-matrix"] h2').should('have.length', MODEL_LABELS.length); + for (const label of MODEL_LABELS) { + cy.get('[data-testid="overview-desktop-matrix"]').should('contain.text', label); + } + // Database-wide freshness line + the methodology footnote's once-per-page note. + cy.contains('Database snapshot through Jul 18').should('exist'); + cy.contains( + 'The default precision maximizes comparable hardware coverage. Not ranked does not mean slower.', + ).should('exist'); + // A one-glance summary, not an interactive widget. + cy.get('[data-testid="overview-desktop-matrix"]').within(() => { + cy.get('details, summary, button').should('not.exist'); + }); + }); + + it('ranks the primary precision, dates each read, and accounts for every hardware', () => { + cy.viewport(1280, 800); + cy.visit('/overview'); + + desktopModel('DeepSeek-V4-Pro').within(() => { + // Primary precision, its leader, and the runner-up's own signed delta (ASCII minus). + cy.contains('FP4 · PRIMARY @50').should('exist'); + cy.contains('B300').should('exist'); + cy.contains('1,122').should('exist'); + cy.contains('Leader').should('exist'); + cy.contains('-20%').should('exist'); + // FP8 measured one platform → one compact coverage line, not a second table. + cy.contains('FP8 coverage: GB200 NVL72 measured; insufficient comparable results.').should( + 'exist', + ); + // Leader @50 is bracketed by two run days (en-dash range); same-day reads and + // the @100 read (its own leader) collapse to a single date. + cy.contains('Jun 24–Jul 4').should('exist'); + cy.contains('Jul 18').should('exist'); + cy.contains('381').should('exist'); + cy.contains('Only exact result').should('exist'); + // Every remaining hardware carries one reason, both clamp directions included. + cy.contains('cannot reach @50').should('exist'); + cy.contains('no exact @50 result').should('exist'); + cy.contains('no 8K/1K data').should('exist'); + cy.contains('standard decode only').should('exist'); + // Each ranked value links into its own pre-filtered dashboard view. + cy.get('a[href*="i_gpus="]') + .first() + .should('have.attr', 'href') + .and('include', 'g_model=DeepSeek-V4-Pro') + .and('include', 'i_gpus=b300_sglang_mtp') + .and('include', 'i_spec=mtp'); + }); + }); + + it('opens a ranked secondary only when the other precision adds comparable hardware', () => { + cy.viewport(1280, 800); + cy.visit('/overview'); + + // Qwen: FP8 ranks a comparable pair AND adds MI355X, so it renders subordinate rows. + desktopModel('Qwen-3.5-397B-A17B').within(() => { + cy.contains('FP4 · PRIMARY @50').should('exist'); + cy.contains('FP8 @50').should('exist'); + cy.contains('MI355X').should('exist'); + cy.contains('760').should('exist'); + cy.contains('-16%').should('exist'); + }); + // MiniMax: wider exact-@50 FP8 coverage flips the primary precision to FP8; + // FP4 falls back to a single-hardware coverage line. + desktopModel('MiniMax-M3').within(() => { + cy.contains('FP8 · PRIMARY @50').should('exist'); + cy.contains('FP4 coverage: H200 measured; insufficient comparable results.').should('exist'); + }); + }); + + it('stacks on a 390px phone with no sideways scroll', () => { + cy.viewport(390, 844); + cy.visit('/overview'); + + cy.get('[data-testid="overview-mobile-list"]').should('be.visible'); + cy.get('[data-testid="overview-desktop-matrix"]').should('not.be.visible'); + cy.get('[data-testid="overview-mobile-list"]').within(() => { + cy.get('details, summary, button').should('not.exist'); + }); + expectNoHorizontalOverflow(); + expectNoHorizontalScroller('overview-mobile-list'); + }); + + it('renders the Chinese sibling with the same hierarchy and the per-GPU unit', () => { + cy.viewport(1280, 800); + cy.visit('/zh/overview'); + + cy.contains('h1', 'AI 推理总览').should('exist'); + cy.contains('输出 tok/s/GPU').should('exist'); + desktopModel('DeepSeek-V4-Pro').within(() => { + cy.contains('FP4 · 主排名 @50').should('exist'); + }); + cy.contains('默认精度优先覆盖更多可比较硬件。未参与排名不代表性能更低。').should('exist'); + }); +}); diff --git a/packages/app/cypress/fixtures/api/overview-rows.json b/packages/app/cypress/fixtures/api/overview-rows.json new file mode 100644 index 00000000..d0e53d8a --- /dev/null +++ b/packages/app/cypress/fixtures/api/overview-rows.json @@ -0,0 +1,26 @@ +{ + "DeepSeek-V4-Pro": [ + {"id":1,"hardware":"b300","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":1300},"date":"2026-06-24","run_url":null}, + {"id":2,"hardware":"b300","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":70,"output_tput_per_gpu":1000},"date":"2026-07-04","run_url":null}, + {"id":3,"hardware":"b200","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":40,"output_tput_per_gpu":1050},"date":"2026-07-18","run_url":null}, + {"id":4,"hardware":"b200","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":24,"offload_mode":"off","image":null,"metrics":{"median_intvty":110,"output_tput_per_gpu":300},"date":"2026-07-18","run_url":null}, + {"id":5,"hardware":"mi355x","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":60,"output_tput_per_gpu":760},"date":"2026-07-18","run_url":null}, + {"id":6,"hardware":"gb300","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":45,"output_tput_per_gpu":1050},"date":"2026-07-18","run_url":null}, + {"id":7,"hardware":"h100","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"none","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":800},"date":"2026-07-18","run_url":null}, + {"id":8,"hardware":"gb200","framework":"sglang","model":"dsv4","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":600},"date":"2026-07-18","run_url":null} + ], + "Qwen-3.5-397B-A17B": [ + {"id":9,"hardware":"b300","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":1400},"date":"2026-07-18","run_url":null}, + {"id":10,"hardware":"b300","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":70,"output_tput_per_gpu":980},"date":"2026-07-18","run_url":null}, + {"id":11,"hardware":"b200","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":40,"output_tput_per_gpu":1150},"date":"2026-07-18","run_url":null}, + {"id":12,"hardware":"b200","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":24,"offload_mode":"off","image":null,"metrics":{"median_intvty":80,"output_tput_per_gpu":720},"date":"2026-07-18","run_url":null}, + {"id":13,"hardware":"b200","framework":"sglang","model":"qwen3.5","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":900},"date":"2026-07-18","run_url":null}, + {"id":14,"hardware":"mi355x","framework":"sglang","model":"qwen3.5","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":760},"date":"2026-07-18","run_url":null} + ], + "MiniMax-M3": [ + {"id":15,"hardware":"h200","framework":"sglang","model":"minimaxm3","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":820},"date":"2026-07-18","run_url":null}, + {"id":16,"hardware":"h200","framework":"sglang","model":"minimaxm3","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":70,"output_tput_per_gpu":560},"date":"2026-07-18","run_url":null}, + {"id":17,"hardware":"mi325x","framework":"sglang","model":"minimaxm3","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":650},"date":"2026-07-18","run_url":null}, + {"id":18,"hardware":"h200","framework":"sglang","model":"minimaxm3","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":600},"date":"2026-07-18","run_url":null} + ] +} diff --git a/packages/app/src/app/overview/page.tsx b/packages/app/src/app/overview/page.tsx new file mode 100644 index 00000000..e76d0d35 --- /dev/null +++ b/packages/app/src/app/overview/page.tsx @@ -0,0 +1,40 @@ +import type { Metadata } from 'next'; + +import { SITE_NAME, SITE_URL } from '@semianalysisai/inferencex-constants'; + +import { OverviewPageContent } from '@/components/overview/overview-page'; +import { enAlternates } from '@/lib/i18n'; +import { getOverviewPageData } from '@/lib/overview-data.server'; + +export const dynamic = 'force-dynamic'; + +const DESCRIPTION = + 'Comparable validated AI inference serving results for every active model and platform at a fixed single-turn 8K input / 1K output workload, ranked at 50 tok/s/user.'; + +export const metadata: Metadata = { + title: 'AI Inference Overview', + description: DESCRIPTION, + alternates: enAlternates('/overview'), + openGraph: { + title: `AI Inference Overview | ${SITE_NAME}`, + description: DESCRIPTION, + url: `${SITE_URL}/overview`, + type: 'website', + }, + twitter: { + card: 'summary_large_image', + title: `AI Inference Overview | ${SITE_NAME}`, + description: DESCRIPTION, + }, +}; + +export default async function OverviewPage() { + const data = await getOverviewPageData(); + return ( +
+
+ +
+
+ ); +} diff --git a/packages/app/src/app/sitemap.ts b/packages/app/src/app/sitemap.ts index 32bc84d4..2dd3e934 100644 --- a/packages/app/src/app/sitemap.ts +++ b/packages/app/src/app/sitemap.ts @@ -58,6 +58,11 @@ export default async function sitemap(): Promise { return [ ...localizedPair('/', { lastModified: now, changeFrequency: 'daily', priority: 1 }), + ...localizedPair('/overview', { + lastModified: now, + changeFrequency: 'daily', + priority: 0.9, + }), ...TABS.flatMap((tab) => localizedPair(`/${tab}`, { lastModified: now, diff --git a/packages/app/src/app/zh/overview/page.tsx b/packages/app/src/app/zh/overview/page.tsx new file mode 100644 index 00000000..91e4fd48 --- /dev/null +++ b/packages/app/src/app/zh/overview/page.tsx @@ -0,0 +1,41 @@ +import type { Metadata } from 'next'; + +import { SITE_NAME, SITE_URL } from '@semianalysisai/inferencex-constants'; + +import { OverviewPageContent } from '@/components/overview/overview-page'; +import { ZH_OG_LOCALE, zhAlternates } from '@/lib/i18n'; +import { getOverviewPageData } from '@/lib/overview-data.server'; + +export const dynamic = 'force-dynamic'; + +const DESCRIPTION = + '以固定单轮 8K 输入 / 1K 输出负载,按 50 tok/s/user 档位排名,对比每个活跃模型在各硬件平台上可比的已验证服务结果。'; + +export const metadata: Metadata = { + title: 'AI 推理总览', + description: DESCRIPTION, + alternates: zhAlternates('/overview'), + openGraph: { + title: `AI 推理总览 | ${SITE_NAME}`, + description: DESCRIPTION, + url: `${SITE_URL}/zh/overview`, + type: 'website', + locale: ZH_OG_LOCALE, + }, + twitter: { + card: 'summary_large_image', + title: `AI 推理总览 | ${SITE_NAME}`, + description: DESCRIPTION, + }, +}; + +export default async function ZhOverviewPage() { + const data = await getOverviewPageData(); + return ( +
+
+ +
+
+ ); +} diff --git a/packages/app/src/components/header/header.tsx b/packages/app/src/components/header/header.tsx index b5fbbf71..09926859 100644 --- a/packages/app/src/components/header/header.tsx +++ b/packages/app/src/components/header/header.tsx @@ -83,7 +83,7 @@ function LanguageToggle({ pathname }: { pathname: string }) { href={target} data-testid="language-toggle" hrefLang={isZh ? 'en' : 'zh-CN'} - className="px-2 py-1.5 rounded-md text-sm font-medium text-muted-foreground hover:text-foreground hover:bg-muted transition-colors whitespace-nowrap" + className="inline-flex items-center min-h-11 px-2 rounded-md text-sm font-medium text-muted-foreground hover:text-foreground hover:bg-muted transition-colors whitespace-nowrap" onClick={() => track('header_language_toggled', { to: isZh ? 'en' : 'zh' })} > {isZh ? 'EN' : '中文'} @@ -145,7 +145,11 @@ export const Header = ({ starCount }: { starCount?: number | null }) => {
{/* Brand */} - + InferenceX by @@ -184,9 +188,18 @@ export const Header = ({ starCount }: { starCount?: number | null }) => { {/* Right side */}
- + + + - + {/* Minecraft-only audio toggles ride alongside GitHub stars below the + same `sm` cutoff: in minecraft mode the pixel font widens the bar, + and these two extra buttons are what push a 320px header past its + bounds. Below `sm` they move into the mobile menu instead (see the + MinecraftToggles instance in the hamburger dropdown below). */} + + + {/* Mobile hamburger */} @@ -195,7 +208,7 @@ export const Header = ({ starCount }: { starCount?: number | null }) => { type="button" data-testid="mobile-menu-toggle" onClick={toggleMenu} - className="flex items-center justify-center size-9 rounded-md transition-colors hover:bg-muted cursor-pointer" + className="flex items-center justify-center size-11 rounded-md transition-colors hover:bg-muted cursor-pointer" aria-expanded={mobileMenuOpen} aria-label="Navigation menu" > @@ -224,7 +237,7 @@ export const Header = ({ starCount }: { starCount?: number | null }) => { key={href} href={displayHref} className={cn( - 'px-3 py-2 rounded-md text-sm font-medium transition-colors', + 'flex items-center min-h-11 px-3 rounded-md text-sm font-medium transition-colors', isActive(pathname, href) ? 'text-brand bg-brand/10' : 'text-muted-foreground hover:text-foreground hover:bg-muted', @@ -237,6 +250,9 @@ export const Header = ({ starCount }: { starCount?: number | null }) => { {label} ))} + + +
)}
diff --git a/packages/app/src/components/overview/overview-detail-link.tsx b/packages/app/src/components/overview/overview-detail-link.tsx new file mode 100644 index 00000000..f7b23298 --- /dev/null +++ b/packages/app/src/components/overview/overview-detail-link.tsx @@ -0,0 +1,86 @@ +'use client'; + +import { ArrowRight } from 'lucide-react'; +import type { ReactNode } from 'react'; + +import { track } from '@/lib/analytics'; +import { cn } from '@/lib/utils'; +import type { OverviewConfigResult, OverviewModelSummary } from '@/lib/overview-data'; +import { buildOverviewDashboardHref } from '@/lib/overview-links'; + +/** + * Model-level dashboard link. A plain anchor for the same reason the evidence + * links below are: the dashboard reads its filters from a snapshot `url-state.ts` + * takes at module evaluation, so a client-side `` navigation would land on + * an unfiltered dashboard while the label promises the model's own view. + */ +export function OverviewDetailLink({ + href, + model, + ariaLabel, + className, + children, +}: { + href: string; + model: string; + ariaLabel: string; + className?: string; + children: ReactNode; +}) { + return ( + track('overview_model_detail_clicked', { model })} + > + {children} + + ); +} + +const EVIDENCE_LINK_CLASS = + 'group inline-flex min-h-11 w-fit items-center gap-1 whitespace-nowrap rounded-sm text-xs text-muted-foreground underline decoration-dotted underline-offset-4 transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 motion-reduce:transition-none'; + +/** + * One evidence link: the /inference dashboard pre-filtered to a ranked + * configuration. The visible text stays short so the compact rows scan; the + * full deployment topology rides in `ariaLabel`, which the caller also makes + * distinct per cohort — so no two evidence links share the accessible name + * "Open filtered dashboard". + * + * A plain anchor, not a ``: the dashboard reads its share-link params + * from a snapshot `url-state.ts` takes at module evaluation, so a client-side + * navigation would arrive with every filter dropped — silently unfiltered while + * the accessible name promises filtered evidence. + */ +export function OverviewDashboardLink({ + locale, + model, + config, + ariaLabel, + children, +}: { + locale: 'en' | 'zh'; + model: OverviewModelSummary; + config: OverviewConfigResult; + ariaLabel: string; + children: ReactNode; +}) { + return ( + + {children} + + ); +} diff --git a/packages/app/src/components/overview/overview-page.tsx b/packages/app/src/components/overview/overview-page.tsx new file mode 100644 index 00000000..c99f3ae6 --- /dev/null +++ b/packages/app/src/components/overview/overview-page.tsx @@ -0,0 +1,59 @@ +import { Card } from '@/components/ui/card'; +import type { OverviewPageData } from '@/lib/overview-data'; + +import { + DesktopOverviewMatrix, + MobileOverviewList, + OverviewMethodology, + overviewFormatters, + OVERVIEW_STRINGS, + type OverviewLocale, +} from './overview-scorecard'; + +interface OverviewPageProps { + data: OverviewPageData; + locale: OverviewLocale; +} + +export function OverviewPageContent({ data, locale }: OverviewPageProps) { + const strings = OVERVIEW_STRINGS[locale]; + const formatters = overviewFormatters(locale); + const snapshot = + data.datasetThroughDate === null + ? null + : strings.snapshot(formatters.shortDate(data.datasetThroughDate)); + + return ( +
+
+

{strings.title}

+

+ {strings.purpose} +

+

+ {strings.scope} +

+ {snapshot === null ? null : ( +

{snapshot}

+ )} +
+ + {/* Official-only summary; uploaded runs remain in the linked dashboard. */} + + + + + +
+ ); +} diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx new file mode 100644 index 00000000..a1c02907 --- /dev/null +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -0,0 +1,736 @@ +/** + * @file overview-scorecard.tsx + * @description The comparison surface itself. Desktop (a matrix at `xl`) and + * mobile (stacked cards below `xl`) render the SAME per-model hierarchy from one + * `ModelHierarchy` component, so the two surfaces cannot drift semantically. + * + * Each model reads top to bottom as: the primary precision's ranked hardware as + * horizontal chips (@50), the secondary precision (a subordinate ranked block or + * one compact coverage line), the @100 capability read, then a not-ranked line + * that accounts for every remaining hardware with an explicit reason. Precisions + * and incompatible cohorts are never compared across — each ranks only itself. + */ + +import { Badge } from '@/components/ui/badge'; +import { + overviewHighValue, + OVERVIEW_HIGH_TIER, + OVERVIEW_PRIMARY_TIER, + type OverviewComparisonGroup, + type OverviewConfigResult, + type OverviewHardwareStatus, + type OverviewModelSummary, + type OverviewNotRankedEntry, +} from '@/lib/overview-data'; +import { detailHref } from '@/lib/overview-links'; + +import { OverviewDetailLink, OverviewDashboardLink } from './overview-detail-link'; + +export type OverviewLocale = 'en' | 'zh'; + +export const OVERVIEW_STRINGS = { + en: { + title: 'AI Inference Overview', + purpose: 'Compare validated serving results across active models and hardware.', + scope: `8K→1K · Single-turn · Output tok/s/GPU at ${OVERVIEW_PRIMARY_TIER} tok/s/user · Speculative decode only · FP4/FP8 by comparable coverage`, + snapshot: (through: string) => `Database snapshot through ${through}`, + modelHeader: 'Model', + resultsHeader: 'Ranked results', + detailsHeader: 'Details', + caption: `Validated output tok/s/GPU per model and comparable cohort at ${OVERVIEW_PRIMARY_TIER} and ${OVERVIEW_HIGH_TIER} tok/s/user.`, + cohort: (engine: string, mode: string) => `${engine} · ${mode}`, + aggregated: 'Aggregated', + disaggregated: 'Disaggregated', + anyEngine: 'All engines', + engineGroupLabels: { + trt: 'TRTLLM family', + sglang: 'SGLang/ATOM family', + vllm: 'vLLM family', + } as Record, + engineGroupFallback: (engineGroup: string) => `${engineGroup.toUpperCase()} family`, + primaryHeading: (precision: string) => `${precision} · PRIMARY @${OVERVIEW_PRIMARY_TIER}`, + secondaryHeading: (precision: string) => `${precision} @${OVERVIEW_PRIMARY_TIER}`, + secondaryCoverage: (precision: string, list: string) => + `${precision} coverage: ${list} measured; insufficient comparable results.`, + highHeading: `@${OVERVIEW_HIGH_TIER}`, + leader: 'Leader', + onlyExactResult: 'Only exact result', + highLeader: (hardware: string, value: string) => `${hardware} · ${value}`, + highLeaderSame: 'Same leader', + leaderChange: 'Leader change', + highInsufficient: `No exact read at ${OVERVIEW_HIGH_TIER} tok/s/user`, + highNoPrimaryBaseline: `No ${OVERVIEW_PRIMARY_TIER} tok/s/user baseline`, + evidenceHighDashboard: `Dashboard @${OVERVIEW_HIGH_TIER}`, + evidenceDashboardAria: (model: string, config: string) => + `Open filtered dashboard: ${model} · ${config}`, + evidenceHighAria: (model: string, config: string) => + `Open filtered dashboard @${OVERVIEW_HIGH_TIER} tok/s/user leader: ${model} · ${config}`, + notRankedLine: (entries: string) => `Not ranked: ${entries}`, + notRankedReasons: { + standard_decode_only: 'standard decode only', + int4_bf16_only: 'INT4/BF16 only', + different_serving_cohort: 'different serving cohort', + no_8k1k_data: 'no 8K/1K data', + cannot_reach_at50: `cannot reach @${OVERVIEW_PRIMARY_TIER}`, + no_exact_at50: `no exact @${OVERVIEW_PRIMARY_TIER} result`, + } as Record, + otherPrecisionReason: (precisions: string[]) => + `${precisions.map((precision) => precision.toUpperCase()).join('/')} only`, + joinList: (labels: string[]) => + labels.length <= 1 + ? (labels[0] ?? '') + : `${labels.slice(0, -1).join(', ')} and ${labels.at(-1) ?? ''}`, + methodologyNote: + 'Precisions and incompatible serving cohorts are ranked separately. The default precision maximizes comparable hardware coverage. Not ranked does not mean slower.', + interpolationNote: + 'Tier values interpolate each configuration’s official Pareto frontier — no extrapolation.', + cohortNote: + 'Engine families with different speculative-decode acceptance forcing, and aggregated versus disaggregated deployments, rank as separate cohorts — never against each other.', + detailLink: 'View details', + detailAria: (modelLabel: string) => `View details: ${modelLabel}`, + }, + zh: { + title: 'AI 推理总览', + purpose: '对比活跃模型在不同硬件上的已验证服务结果。', + scope: `8K→1K · 单轮 · ${OVERVIEW_PRIMARY_TIER} tok/s/user 下的输出 tok/s/GPU · 仅投机解码 · 按可比覆盖选择 FP4/FP8`, + snapshot: (through: string) => `数据库快照截至 ${through}`, + modelHeader: '模型', + resultsHeader: '排名结果', + detailsHeader: '详情', + caption: `各模型与可对比分组在 ${OVERVIEW_PRIMARY_TIER} 与 ${OVERVIEW_HIGH_TIER} tok/s/user 下的已验证输出 tok/s/GPU。`, + cohort: (engine: string, mode: string) => `${engine} · ${mode}`, + aggregated: '聚合部署', + disaggregated: '分离部署', + anyEngine: '全部引擎', + engineGroupLabels: { + trt: 'TRTLLM 系列', + sglang: 'SGLang/ATOM 系列', + vllm: 'vLLM 系列', + } as Record, + engineGroupFallback: (engineGroup: string) => `${engineGroup.toUpperCase()} 系列`, + primaryHeading: (precision: string) => `${precision} · 主排名 @${OVERVIEW_PRIMARY_TIER}`, + secondaryHeading: (precision: string) => `${precision} @${OVERVIEW_PRIMARY_TIER}`, + secondaryCoverage: (precision: string, list: string) => + `${precision} 覆盖:已测量 ${list};可比结果不足。`, + highHeading: `@${OVERVIEW_HIGH_TIER}`, + leader: '领先', + onlyExactResult: '唯一精确读数', + highLeader: (hardware: string, value: string) => `${hardware} · ${value}`, + highLeaderSame: '领先者相同', + leaderChange: '领先者变化', + highInsufficient: `${OVERVIEW_HIGH_TIER} tok/s/user 下无精确读数`, + highNoPrimaryBaseline: `无 ${OVERVIEW_PRIMARY_TIER} tok/s/user 基线`, + evidenceHighDashboard: `仪表板 @${OVERVIEW_HIGH_TIER}`, + evidenceDashboardAria: (model: string, config: string) => + `打开筛选后的仪表板:${model} · ${config}`, + evidenceHighAria: (model: string, config: string) => + `打开筛选后的仪表板 @${OVERVIEW_HIGH_TIER} tok/s/user:${model} · ${config}`, + notRankedLine: (entries: string) => `未参与排名:${entries}`, + notRankedReasons: { + standard_decode_only: '仅标准解码', + int4_bf16_only: '仅 INT4/BF16', + different_serving_cohort: '不同服务配置', + no_8k1k_data: '无 8K/1K 数据', + cannot_reach_at50: `无法达到 @${OVERVIEW_PRIMARY_TIER}`, + no_exact_at50: `无精确 @${OVERVIEW_PRIMARY_TIER} 结果`, + } as Record, + otherPrecisionReason: (precisions: string[]) => + `仅 ${precisions.map((precision) => precision.toUpperCase()).join('/')}`, + joinList: (labels: string[]) => labels.join('、'), + methodologyNote: + '不同精度和不可直接比较的服务配置将分别排名。默认精度优先覆盖更多可比较硬件。未参与排名不代表性能更低。', + interpolationNote: '各档位数据基于各配置官方 Pareto 前沿插值;不进行外推。', + cohortNote: + '投机解码接受率强制方式不同的引擎系列,以及聚合与分离部署,按独立分组排名,彼此之间不作比较。', + detailLink: '查看详情', + detailAria: (modelLabel: string) => `查看详情:${modelLabel}`, + }, +} as const; + +export type OverviewStrings = (typeof OVERVIEW_STRINGS)[OverviewLocale]; + +interface Formatters { + number: Intl.NumberFormat; + signed: Intl.NumberFormat; + date: (date: string) => string; + shortDate: (date: string) => string; +} + +export function overviewFormatters(locale: OverviewLocale): Formatters { + const tag = locale === 'zh' ? 'zh-CN' : 'en-US'; + const dateFormat = new Intl.DateTimeFormat(tag, { + year: 'numeric', + month: 'short', + day: 'numeric', + timeZone: 'UTC', + }); + const shortDateFormat = new Intl.DateTimeFormat(tag, { + month: 'short', + day: 'numeric', + timeZone: 'UTC', + }); + return { + number: new Intl.NumberFormat(tag, { maximumFractionDigits: 0 }), + signed: new Intl.NumberFormat(tag, { maximumFractionDigits: 0, signDisplay: 'exceptZero' }), + date: (date) => dateFormat.format(new Date(`${date}T00:00:00Z`)), + shortDate: (date) => shortDateFormat.format(new Date(`${date}T00:00:00Z`)), + }; +} + +/** + * A result's evidence dates, localized: a single day (`Jul 6` / `7月6日`) when + * both backing frontier points share a date, else an en-dash range + * (`Jun 24–Jul 4`). Null when the value has no backing date. + */ +function formatEvidenceDate( + formatters: Formatters, + evidenceDate: { from: string; to: string } | null, +): string | null { + if (evidenceDate === null) return null; + const from = formatters.shortDate(evidenceDate.from); + return evidenceDate.from === evidenceDate.to + ? from + : `${from}–${formatters.shortDate(evidenceDate.to)}`; +} + +function cohortLabel( + group: OverviewComparisonGroup, + strings: OverviewStrings, + showDbModel: boolean, +): string { + const engine = + group.engineGroup === null + ? strings.anyEngine + : (strings.engineGroupLabels[group.engineGroup] ?? + strings.engineGroupFallback(group.engineGroup)); + const mode = + group.deploymentMode === 'disaggregated' ? strings.disaggregated : strings.aggregated; + const label = strings.cohort(engine, mode); + // Point releases of one display model rank in separate cohorts; name the raw + // db model so two same-engine, same-mode cohorts are told apart. + return showDbModel ? `${label} · ${group.dbModel}` : label; +} + +/** Cohorts with at least one exact @50 read — the ones that render ranked chips. */ +function rankedCohorts(groups: readonly OverviewComparisonGroup[]): OverviewComparisonGroup[] { + return groups.filter((group) => group.primaryRanking.state !== 'insufficient_coverage'); +} + +/** A cohort's hardware that hold an exact @50 read, leader first then value desc. */ +function rankedStatuses(group: OverviewComparisonGroup): OverviewHardwareStatus[] { + return group.hardwareStatuses + .filter((status) => status.isPrimaryLeader || status.primaryDeltaPercent !== null) + .toSorted((a, b) => (b.primary.value ?? -1) - (a.primary.value ?? -1)); +} + +/** Not-ranked reason copy; `other_precision_only` names the measured precisions. */ +function notRankedReason(entry: OverviewNotRankedEntry, strings: OverviewStrings): string { + return entry.reason === 'other_precision_only' + ? strings.otherPrecisionReason(entry.precisions) + : strings.notRankedReasons[entry.reason]; +} + +/** The exact deployment topology of a ranked configuration, for an aria-label. */ +function evidenceConfigLabel( + config: OverviewConfigResult, + strings: OverviewStrings, + showDbModel: boolean, +): string { + return [ + config.hardwareLabel, + config.precision.toUpperCase(), + config.frameworkLabel, + config.specLabel, + config.disagg ? strings.disaggregated : strings.aggregated, + config.parallelism, + `${config.totalGpu} GPU`, + ...(showDbModel ? [config.dbModel] : []), + ].join(' · '); +} + +const BLOCK_HEADING_CLASS = + 'mb-0.5 text-xs font-semibold uppercase tracking-wider text-muted-foreground'; +const SUBORDINATE_HEADING_CLASS = + 'mb-0.5 text-[11px] font-medium uppercase tracking-wider text-muted-foreground/80'; + +/** + * One ranked hardware: its label, its @50 value linking to the filtered + * dashboard, and its standing — `Leader`, its signed delta against the leader, + * or `Only exact result` when it is the cohort's lone exact read. + */ +function RankedChip({ + locale, + model, + group, + status, + multiDbModel, + formatters, + strings, +}: { + locale: OverviewLocale; + model: OverviewModelSummary; + group: OverviewComparisonGroup; + status: OverviewHardwareStatus; + multiDbModel: boolean; + formatters: Formatters; + strings: OverviewStrings; +}) { + const { value, config } = status.primary; + const standing = status.isPrimaryLeader + ? group.primaryRanking.state === 'single_measured' + ? strings.onlyExactResult + : strings.leader + : `${formatters.signed.format(status.primaryDeltaPercent as number)}%`; + const date = formatEvidenceDate(formatters, status.primary.evidenceDate); + return ( +

+ {status.hardwareLabel} + {value === null ? null : config === null ? ( + + {formatters.number.format(value)} + + ) : ( + + + {formatters.number.format(value)} + + + )} + · {standing} + {date === null ? null : ( + · {date} + )} +

+ ); +} + +/** A precision's ranked cohorts as stacked chips, under one heading. */ +function RankedSection({ + locale, + model, + cohorts, + heading, + headingClass, + formatters, + strings, +}: { + locale: OverviewLocale; + model: OverviewModelSummary; + cohorts: OverviewComparisonGroup[]; + heading: string; + headingClass: string; + formatters: Formatters; + strings: OverviewStrings; +}) { + if (cohorts.length === 0) return null; + const multiDbModel = new Set(cohorts.map((group) => group.dbModel)).size > 1; + return ( +
+

{heading}

+ {cohorts.map((group) => ( +
+ {cohorts.length > 1 ? ( +

+ {cohortLabel(group, strings, multiDbModel)} +

+ ) : null} + {rankedStatuses(group).map((status) => ( + + ))} +
+ ))} +
+ ); +} + +/** The primary precision block: its heading names the precision and the tier. */ +function PrimarySection({ + locale, + model, + cohorts, + formatters, + strings, +}: { + locale: OverviewLocale; + model: OverviewModelSummary; + cohorts: OverviewComparisonGroup[]; + formatters: Formatters; + strings: OverviewStrings; +}) { + const precision = (model.selectedPrecision ?? '').toUpperCase(); + return ( + + ); +} + +/** + * The secondary precision, subordinate to the primary: a ranked block when it + * genuinely adds comparable hardware, otherwise one compact coverage line so the + * precision still surfaces without a second full table. + */ +function SecondarySection({ + locale, + model, + formatters, + strings, +}: { + locale: OverviewLocale; + model: OverviewModelSummary; + formatters: Formatters; + strings: OverviewStrings; +}) { + const secondary = model.secondary; + if (secondary === null) return null; + const precision = secondary.precision.toUpperCase(); + if (secondary.state === 'coverage') { + const list = strings.joinList(secondary.measuredHardware); + if (list === '') return null; + return ( +

{strings.secondaryCoverage(precision, list)}

+ ); + } + return ( + + ); +} + +/** + * The 100 tok/s/user read of one cohort. It is ranked from its own evidence, so + * it names its own leader and says plainly whether that leader changed. + */ +function HighTierRead({ + group, + formatters, + strings, +}: { + group: OverviewComparisonGroup; + formatters: Formatters; + strings: OverviewStrings; +}) { + const leader = group.highRanking.leader; + const value = leader === null ? null : overviewHighValue(leader); + if (leader === null || value === null) { + return {strings.highInsufficient}; + } + // The @100 leader's own read date — from its hardware's high read, so a @100 + // read backed by a different config than the @50 leader shows its own date. + const date = formatEvidenceDate( + formatters, + group.hardwareStatuses.find((status) => status.hardware === leader.hardware)?.high + .evidenceDate ?? null, + ); + return ( + + + {strings.highLeader(leader.hardwareLabel, formatters.number.format(value))} + + {group.highRanking.state === 'single_measured' ? ( + {strings.onlyExactResult} + ) : group.highLeaderTransition === 'changed_hardware' ? ( + + {strings.leaderChange} + + ) : group.highLeaderTransition === 'no_primary_baseline' ? ( + {strings.highNoPrimaryBaseline} + ) : ( + {strings.highLeaderSame} + )} + {date === null ? null : ( + · {date} + )} + + ); +} + +/** + * The @100 capability read per comparable cohort. When the @100 leader runs a + * different configuration than the @50 leader, its own filtered dashboard rides + * along so the high-tier read ships with the evidence behind it. + */ +function HighSection({ + locale, + model, + cohorts, + formatters, + strings, +}: { + locale: OverviewLocale; + model: OverviewModelSummary; + cohorts: OverviewComparisonGroup[]; + formatters: Formatters; + strings: OverviewStrings; +}) { + const highCohorts = cohorts.filter((group) => group.primaryRanking.state === 'comparable'); + if (highCohorts.length === 0) return null; + const multiDbModel = new Set(highCohorts.map((group) => group.dbModel)).size > 1; + return ( +
+

{strings.highHeading}

+ {highCohorts.map((group) => { + const primaryLeader = group.primaryRanking.leader; + const highLeader = group.highRanking.leader; + return ( +
+ {highCohorts.length > 1 ? ( + + {cohortLabel(group, strings, multiDbModel)} + + ) : null} + + {highLeader !== null && + (primaryLeader === null || highLeader.key !== primaryLeader.key) ? ( + + {strings.evidenceHighDashboard} + + ) : null} +
+ ); + })} +
+ ); +} + +/** + * Every remaining hardware, accounted for in one line: those with no exact @50 + * read and not ranked by the secondary block, each with its explicit reason. The + * standing note "not ranked does not mean slower" lives once in the methodology. + */ +function NotRankedLine({ + model, + strings, +}: { + model: OverviewModelSummary; + strings: OverviewStrings; +}) { + // Hardware already named by the secondary section — its ranked rows or its + // coverage 'measured' list — is not repeated here; dedup against the secondary's + // measured labels so each hardware surfaces in exactly one place. + const secondaryMeasured = new Set(model.secondary?.measuredHardware); + const entries = model.notRanked + .filter((entry) => !secondaryMeasured.has(entry.hardwareLabel)) + .map((entry) => `${entry.hardwareLabel} — ${notRankedReason(entry, strings)}`) + .join(' · '); + if (entries === '') return null; + return

{strings.notRankedLine(entries)}

; +} + +/** The full per-model hierarchy, identical on both surfaces. */ +function ModelHierarchy({ + locale, + model, + formatters, + strings, +}: { + locale: OverviewLocale; + model: OverviewModelSummary; + formatters: Formatters; + strings: OverviewStrings; +}) { + const cohorts = rankedCohorts(model.comparisonGroups); + return ( +
+ + + + +
+ ); +} + +/** + * The model's name and — for a coverage-only model that shows no ranked chips — + * its freshness. A ranked model dates each result on its own chip instead, so + * the model-level date is only the fallback for a model with nothing to rank. + */ +function ModelIdentity({ + model, + formatters, +}: { + model: OverviewModelSummary; + formatters: Formatters; +}) { + const hasRankedResults = + rankedCohorts(model.comparisonGroups).length > 0 || model.secondary?.state === 'ranked'; + const freshness = + hasRankedResults || model.latestWorkloadDate === null + ? null + : formatters.date(model.latestWorkloadDate); + return ( +
+

{model.modelLabel}

+ {freshness ? ( + {freshness} + ) : null} +
+ ); +} + +interface SurfaceProps { + models: OverviewModelSummary[]; + locale: OverviewLocale; + formatters: Formatters; + strings: OverviewStrings; +} + +/** Model column plus one content column carrying the full hierarchy; no scroll. */ +export function DesktopOverviewMatrix({ models, locale, formatters, strings }: SurfaceProps) { + return ( +
+ + + + + + + + + + + + + + + {models.map((model) => ( + + + + + + + + ))} +
{strings.caption}
+ {strings.modelHeader} + + {strings.resultsHeader} + + {strings.detailsHeader} +
+ + + + + + {strings.detailLink} + +
+
+ ); +} + +/** Below `xl`: the same hierarchy stacked as cards, always fully visible. */ +export function MobileOverviewList({ models, locale, formatters, strings }: SurfaceProps) { + return ( +
    + {models.map((model) => ( +
  • +
    + + + + {strings.detailLink} + +
    +
  • + ))} +
+ ); +} + +export function OverviewMethodology({ strings }: { strings: OverviewStrings }) { + return ( +
+

{strings.methodologyNote}

+

{strings.cohortNote}

+

{strings.interpolationNote}

+
+ ); +} diff --git a/packages/app/src/components/ui/mode-toggle.tsx b/packages/app/src/components/ui/mode-toggle.tsx index f216624f..7dcdd354 100644 --- a/packages/app/src/components/ui/mode-toggle.tsx +++ b/packages/app/src/components/ui/mode-toggle.tsx @@ -27,7 +27,7 @@ export function ModeToggle() { }; const buttonClasses = cn( - 'inline-flex items-center justify-center rounded-md p-2', + 'inline-flex items-center justify-center rounded-md size-11', 'text-muted-foreground hover:text-foreground hover:bg-accent', 'transition-colors duration-200', 'focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2', diff --git a/packages/app/src/hooks/useTogglableSet.ts b/packages/app/src/hooks/useTogglableSet.ts index bf84694c..59bbac71 100644 --- a/packages/app/src/hooks/useTogglableSet.ts +++ b/packages/app/src/hooks/useTogglableSet.ts @@ -1,33 +1,8 @@ import { useCallback, useState } from 'react'; -/** - * Pure toggle state transition. - * - * - If all items are active and one is toggled, only that item remains active (solo). - * - If only one item is active and it's toggled, all items become active (restore all). - * - Otherwise, the item is simply added or removed from the active set. - */ -export function computeToggle(prev: Set, item: string, allItems: Set): Set { - const allAreActive = prev.size === allItems.size; - const isActive = prev.has(item); +import { computeToggle } from '@/lib/toggle-set'; - if (isActive) { - if (allAreActive) { - // Solo: only keep the clicked item - return new Set([item]); - } else if (prev.size === 1) { - // Restore all: re-enable everything - return allItems; - } - // Remove the clicked item - const next = new Set(prev); - next.delete(item); - return next; - } - // Add the clicked item - const next = new Set([...prev, item]); - return next; -} +export { computeToggle }; /** * Hook for managing a togglable set with "click to solo, click again to restore all" behavior. diff --git a/packages/app/src/lib/benchmark-data.server.ts b/packages/app/src/lib/benchmark-data.server.ts new file mode 100644 index 00000000..ccc85317 --- /dev/null +++ b/packages/app/src/lib/benchmark-data.server.ts @@ -0,0 +1,21 @@ +import { FIXTURES_MODE, getDb } from '@semianalysisai/inferencex-db/connection'; +import { + type BenchmarkRow, + getLatestBenchmarks, +} from '@semianalysisai/inferencex-db/queries/benchmarks'; + +import { cachedQuery } from '@/lib/api-cache'; +import { loadFixture } from '@/lib/test-fixtures'; + +/** Cache slot is keyed on the dbKeys array. Both `/compare/` and + * `/compare-per-dollar/` for the same model hit the same blob entry — + * the per-dollar route doesn't duplicate the fetch or the cache. */ +export const getCachedBenchmarks = cachedQuery( + (dbModelKeys: string[]) => { + if (FIXTURES_MODE) return Promise.resolve(loadFixture('benchmarks')); + + return getLatestBenchmarks(getDb(), dbModelKeys); + }, + 'benchmarks', + { blobOnly: true }, +); diff --git a/packages/app/src/lib/compare-ssr.ts b/packages/app/src/lib/compare-ssr.ts index 592a8238..5a3ec2c6 100644 --- a/packages/app/src/lib/compare-ssr.ts +++ b/packages/app/src/lib/compare-ssr.ts @@ -18,16 +18,11 @@ import { SITE_URL, sequenceToIslOsl, } from '@semianalysisai/inferencex-constants'; -import { FIXTURES_MODE, getDb } from '@semianalysisai/inferencex-db/connection'; - -import { - type BenchmarkRow, - getLatestBenchmarks, -} from '@semianalysisai/inferencex-db/queries/benchmarks'; +import type { BenchmarkRow } from '@semianalysisai/inferencex-db/queries/benchmarks'; import { interpolateForGPU } from '@/components/calculator/interpolation'; import type { GPUDataPoint, InterpolatedResult } from '@/components/calculator/types'; -import { cachedQuery } from '@/lib/api-cache'; +import { getCachedBenchmarks } from '@/lib/benchmark-data.server'; import { rowToAggDataEntry } from '@/lib/benchmark-transform'; import { getHardwareKey } from '@/lib/chart-utils'; import { @@ -39,24 +34,8 @@ import { compareModelSeoName, } from '@/lib/compare-slug'; import { getHardwareConfig, getGpuSpecs } from '@/lib/constants'; -import { loadFixture } from '@/lib/test-fixtures'; - -// --------------------------------------------------------------------------- -// Cached benchmark fetch -// --------------------------------------------------------------------------- -/** Cache slot is keyed on the dbKeys array. Both `/compare/` and - * `/compare-per-dollar/` for the same model hit the same blob entry — - * the per-dollar route doesn't duplicate the fetch or the cache. */ -export const getCachedBenchmarks = cachedQuery( - (dbModelKeys: string[]) => { - if (FIXTURES_MODE) return Promise.resolve(loadFixture('benchmarks')); - - return getLatestBenchmarks(getDb(), dbModelKeys); - }, - 'benchmarks', - { blobOnly: true }, -); +export { getCachedBenchmarks }; // --------------------------------------------------------------------------- // URL-param validators (shared by both routes' overrides) diff --git a/packages/app/src/lib/exclusion.ts b/packages/app/src/lib/exclusion.ts index 21f4d1bd..941202f1 100644 --- a/packages/app/src/lib/exclusion.ts +++ b/packages/app/src/lib/exclusion.ts @@ -1,6 +1,6 @@ import { SPEC_METHOD_KEYS } from '@semianalysisai/inferencex-constants'; -import { computeToggle } from '@/hooks/useTogglableSet'; +import { computeToggle } from '@/lib/toggle-set'; /** * Data-driven config exclusion. diff --git a/packages/app/src/lib/i18n.test.ts b/packages/app/src/lib/i18n.test.ts index 492d2742..36a9fde3 100644 --- a/packages/app/src/lib/i18n.test.ts +++ b/packages/app/src/lib/i18n.test.ts @@ -40,6 +40,7 @@ describe('hasZhSibling', () => { it('matches mirrored exact routes', () => { expect(hasZhSibling('/')).toBe(true); expect(hasZhSibling('/inference')).toBe(true); + expect(hasZhSibling('/overview')).toBe(true); expect(hasZhSibling('/about')).toBe(true); }); @@ -72,12 +73,14 @@ describe('switchLocalePath', () => { it('switches English pages to their zh sibling', () => { expect(switchLocalePath('/')).toBe('/zh'); expect(switchLocalePath('/inference')).toBe('/zh/inference'); + expect(switchLocalePath('/overview')).toBe('/zh/overview'); expect(switchLocalePath('/blog/some-post')).toBe('/zh/blog/some-post'); }); it('switches zh pages back to English', () => { expect(switchLocalePath('/zh')).toBe('/'); expect(switchLocalePath('/zh/quotes')).toBe('/quotes'); + expect(switchLocalePath('/zh/overview')).toBe('/overview'); expect(switchLocalePath('/zh/blog/some-post')).toBe('/blog/some-post'); }); diff --git a/packages/app/src/lib/i18n.ts b/packages/app/src/lib/i18n.ts index b094b456..7577353b 100644 --- a/packages/app/src/lib/i18n.ts +++ b/packages/app/src/lib/i18n.ts @@ -36,6 +36,7 @@ export function isZhPathname(pathname: string): boolean { */ export const ZH_MIRRORED_ROUTES: readonly { path: string; exact?: boolean }[] = [ { path: '/', exact: true }, + { path: '/overview', exact: true }, { path: '/inference', exact: true }, { path: '/inference/agentic' }, { path: '/evaluation', exact: true }, diff --git a/packages/app/src/lib/overview-config-identity.test.ts b/packages/app/src/lib/overview-config-identity.test.ts new file mode 100644 index 00000000..02fab087 --- /dev/null +++ b/packages/app/src/lib/overview-config-identity.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; + +import type { BenchmarkRow } from './api'; +import { overviewConfigIdentityKey } from './overview-config-identity'; + +let nextId = 1; + +function row(overrides: Partial = {}): BenchmarkRow { + return { + id: nextId++, + hardware: 'b200', + framework: 'sglang', + model: 'qwen3.5', + precision: 'fp8', + spec_method: 'mtp', + disagg: false, + is_multinode: false, + prefill_tp: 8, + prefill_ep: 1, + prefill_dp_attention: false, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 1, + decode_dp_attention: false, + decode_num_workers: 1, + num_prefill_gpu: 8, + // Not 8: the num_decode_gpu case below flips this to 8 and must change the key. + num_decode_gpu: 4, + benchmark_type: 'single_turn', + isl: 8192, + osl: 1024, + conc: 16, + offload_mode: 'off', + image: null, + metrics: { median_intvty: 50, output_tput_per_gpu: 1000 }, + date: '2026-07-20', + run_url: null, + ...overrides, + }; +} + +/** The pre-v2.1 coarse key: hardware/framework/spec/disagg/offload, no GPU counts. */ +const oldCoarseKey = (r: BenchmarkRow): string => + `${r.hardware}|${r.framework}|${r.spec_method}|${r.disagg}|${r.offload_mode}`; + +describe('overviewConfigIdentityKey', () => { + it.each([ + ['model', 'kimik2.7-code'], + ['hardware', 'gb300'], + ['framework', 'dynamo-trt'], + ['precision', 'fp4'], + ['spec_method', 'eagle'], + ['disagg', true], + ['is_multinode', true], + ['prefill_tp', 4], + ['prefill_ep', 4], + ['prefill_dp_attention', true], + ['prefill_num_workers', 7], + ['decode_tp', 16], + ['decode_ep', 16], + ['decode_dp_attention', true], + ['decode_num_workers', 2], + ['num_prefill_gpu', 28], + ['num_decode_gpu', 8], + ['offload_mode', 'cpu'], + ] satisfies [keyof BenchmarkRow, BenchmarkRow[keyof BenchmarkRow]][])( + 'changes when %s changes', + (field, value) => { + expect(overviewConfigIdentityKey(row({ [field]: value } as Partial))).not.toBe( + overviewConfigIdentityKey(row()), + ); + }, + ); + + it('keeps concurrency and evidence outside configuration identity', () => { + const base = row(); + expect( + overviewConfigIdentityKey({ + ...base, + conc: 64, + date: '2026-07-21', + image: 'new-image', + run_url: 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/2', + }), + ).toBe(overviewConfigIdentityKey(base)); + }); + + it('separates disagg topologies the old coarse key merged (28P+8D vs 40P+16D)', () => { + // The old coarse key omitted GPU counts, so two different P/D splits on the + // same hardware/framework/spec/disagg/offload collided into one result. + const split28p8d = row({ disagg: true, num_prefill_gpu: 28, num_decode_gpu: 8 }); + const split40p16d = row({ disagg: true, num_prefill_gpu: 40, num_decode_gpu: 16 }); + + expect(oldCoarseKey(split28p8d)).toBe(oldCoarseKey(split40p16d)); + expect(overviewConfigIdentityKey(split28p8d)).not.toBe(overviewConfigIdentityKey(split40p16d)); + }); +}); diff --git a/packages/app/src/lib/overview-config-identity.ts b/packages/app/src/lib/overview-config-identity.ts new file mode 100644 index 00000000..18e4e9f4 --- /dev/null +++ b/packages/app/src/lib/overview-config-identity.ts @@ -0,0 +1,33 @@ +import type { BenchmarkRow } from './api'; + +/** + * Exact per-deployment identity for the overview grouping. Every dimension that + * makes a serving configuration a distinct deployable topology is part of the + * key — model, hardware, framework, precision, spec method, disagg, multinode, + * per-role parallelism and worker counts, GPU counts, and offload mode. Rows + * that differ only in concurrency, date, image, or run URL share one identity. + * + * Serialized as a JSON tuple so no field delimiter can collide with a value. + */ +export function overviewConfigIdentityKey(row: BenchmarkRow): string { + return JSON.stringify([ + row.model, + row.hardware, + row.framework, + row.precision, + row.spec_method, + row.disagg, + row.is_multinode, + row.prefill_tp, + row.prefill_ep, + row.prefill_dp_attention, + row.prefill_num_workers, + row.decode_tp, + row.decode_ep, + row.decode_dp_attention, + row.decode_num_workers, + row.num_prefill_gpu, + row.num_decode_gpu, + row.offload_mode ?? 'off', + ]); +} diff --git a/packages/app/src/lib/overview-data.server.ts b/packages/app/src/lib/overview-data.server.ts new file mode 100644 index 00000000..a9d3cafa --- /dev/null +++ b/packages/app/src/lib/overview-data.server.ts @@ -0,0 +1,29 @@ +import { DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; +import { FIXTURES_MODE } from '@semianalysisai/inferencex-db/connection'; + +import type { BenchmarkRow } from '@/lib/api'; +import { getCachedBenchmarks } from '@/lib/benchmark-data.server'; +import { DEFAULT_MODELS } from '@/lib/data-mappings'; +import { assembleOverviewPageData, type OverviewPageData } from '@/lib/overview-data'; +import { loadFixture } from '@/lib/test-fixtures'; + +export async function getOverviewPageData(): Promise { + // E2E fixtures mode serves a small synthetic rows-by-display-model fixture + // through the same assembler the live path uses, so a contract drift breaks + // the fixture tests instead of silently stranding the page's data. + if (FIXTURES_MODE) { + return assembleOverviewPageData(loadFixture>('overview-rows')); + } + + // Fetch rows per db model, concatenated per display model (one display model + // can span several db buckets), then hand the same shape to the assembler. + const entries = await Promise.all( + [...DEFAULT_MODELS].map(async (model) => { + const keys = DISPLAY_MODEL_TO_DB[model] ?? []; + const rows = keys.length > 0 ? await getCachedBenchmarks(keys) : []; + return [model, rows] as const; + }), + ); + + return assembleOverviewPageData(Object.fromEntries(entries)); +} diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts new file mode 100644 index 00000000..96df49f9 --- /dev/null +++ b/packages/app/src/lib/overview-data.test.ts @@ -0,0 +1,740 @@ +import { describe, expect, it } from 'vitest'; + +import overviewRowsFixture from '../../cypress/fixtures/api/overview-rows.json'; + +import type { BenchmarkRow } from './api'; +import { getHardwareConfig } from './constants'; +import { DEFAULT_MODELS, Model, Precision } from './data-mappings'; +import { overviewConfigIdentityKey } from './overview-config-identity'; +import { + assembleOverviewPageData, + buildOverviewHardwareOrder, + buildOverviewModelSummary, + overviewPrimaryValue, + selectOverviewPrecision, + type OverviewModelSummary, +} from './overview-data'; + +let nextId = 1; + +function row(overrides: Partial = {}): BenchmarkRow { + return { + id: nextId++, + hardware: 'b200', + framework: 'sglang', + model: 'qwen3.5', + precision: 'fp8', + spec_method: 'mtp', + disagg: false, + is_multinode: false, + prefill_tp: 8, + prefill_ep: 1, + prefill_dp_attention: false, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 1, + decode_dp_attention: false, + decode_num_workers: 1, + num_prefill_gpu: 8, + num_decode_gpu: 8, + benchmark_type: 'single_turn', + isl: 8192, + osl: 1024, + conc: 16, + offload_mode: 'off', + image: null, + metrics: { median_intvty: 50, output_tput_per_gpu: 1000 }, + date: '2026-07-20', + run_url: null, + ...overrides, + }; +} + +/** One frontier point per tier for a single configuration. */ +function frontier( + throughputs: [number, number, number, number], + overrides: Partial = {}, +): BenchmarkRow[] { + return [30, 50, 75, 100].map((tier, index) => + row({ + conc: index + 1, + metrics: { median_intvty: tier, output_tput_per_gpu: throughputs[index] }, + ...overrides, + }), + ); +} + +/** Frontier at explicit [interactivity, throughput] knots — for clamped/unreachable tiers. */ +function frontierAt( + points: [number, number][], + overrides: Partial = {}, +): BenchmarkRow[] { + return points.map(([intvty, tput], index) => + row({ + conc: index + 1, + metrics: { median_intvty: intvty, output_tput_per_gpu: tput }, + ...overrides, + }), + ); +} + +/** One config's frontier at explicit [interactivity, throughput, date] knots. */ +function datedFrontier( + knots: [number, number, string][], + overrides: Partial = {}, +): BenchmarkRow[] { + return knots.map(([intvty, tput, date], index) => + row({ + conc: index + 1, + metrics: { median_intvty: intvty, output_tput_per_gpu: tput }, + date, + ...overrides, + }), + ); +} + +function statusOf(summary: OverviewModelSummary, hardware: string) { + return summary.hardwareStatuses.find((status) => status.hardware === hardware); +} + +/** + * Leader of the model's first comparable cohort. Leadership only ever exists + * inside a cohort, so every ranking assertion reads it from there — there is no + * model-global winner to compare across engine families or deployment modes. + */ +function primaryLeaderOf(summary: OverviewModelSummary) { + return summary.comparisonGroups[0]?.primaryRanking.leader ?? null; +} + +describe('selectOverviewPrecision', () => { + it('returns null without any FP4/FP8 rows', () => { + expect(selectOverviewPrecision(Model.Qwen3_5, [row({ precision: Precision.BF16 })])).toBeNull(); + expect(selectOverviewPrecision(Model.Qwen3_5, [row({ precision: Precision.INT4 })])).toBeNull(); + }); + + it('ignores standard-decode rows when counting exact-@50 coverage', () => { + // FP8 has the most curves but every one is standard decode; FP4's single + // speculative curve is the only exact-@50 coverage, so it wins outnumbered. + const rows = [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { + hardware: 'gb200', + precision: Precision.FP8, + spec_method: 'none', + }), + ...frontier([1000, 800, 600, 400], { + hardware: 'gb300', + precision: Precision.FP8, + spec_method: 'none', + }), + ]; + + expect(selectOverviewPrecision(Model.Qwen3_5, rows)).toBe(Precision.FP4); + }); +}); + +describe('buildOverviewModelSummary', () => { + it('keeps Kimi point releases as separate exact results, never one blended frontier', () => { + // kimik2.5 and kimik2.7-code both map to Model.Kimi_K2_5. Otherwise + // identical (same hardware/date/config), a model-blind key would merge them + // into one frontier — kimik2.5's 100 tok/s/user point would make the winner + // reachable there. Exact identity keeps kimik2.7-code's frontier pure. + const summary = buildOverviewModelSummary(Model.Kimi_K2_5, [ + row({ + model: 'kimik2.7-code', + conc: 1, + metrics: { median_intvty: 30, output_tput_per_gpu: 1200 }, + }), + row({ + model: 'kimik2.7-code', + conc: 2, + metrics: { median_intvty: 50, output_tput_per_gpu: 1000 }, + }), + row({ + model: 'kimik2.7-code', + conc: 3, + metrics: { median_intvty: 75, output_tput_per_gpu: 800 }, + }), + row({ model: 'kimik2.5', conc: 4, metrics: { median_intvty: 50, output_tput_per_gpu: 400 } }), + row({ + model: 'kimik2.5', + conc: 5, + metrics: { median_intvty: 100, output_tput_per_gpu: 700 }, + }), + ]); + + const leader = primaryLeaderOf(summary); + + expect(leader?.dbModel).toBe('kimik2.7-code'); + expect(overviewPrimaryValue(leader!)).toBe(1000); + expect(leader?.tierValues.find(({ tier }) => tier === 100)).toEqual({ + tier: 100, + value: null, + boundary: 'unreachable', + evidenceDate: null, + }); + }); + + it('splits cohorts by db model so point releases never rank against each other', () => { + // Same hardware, engine group and deployment mode; only the db model + // differs, so the two point releases must land in separate cohorts. + const summary = buildOverviewModelSummary(Model.Kimi_K2_5, [ + ...frontier([1200, 1000, 800, 600], { model: 'kimik2.5' }), + ...frontier([1100, 900, 700, 500], { model: 'kimik2.7-code' }), + ]); + + expect(summary.comparisonGroups.map((group) => group.dbModel).toSorted()).toEqual([ + 'kimik2.5', + 'kimik2.7-code', + ]); + }); + + it('never compares incompatible DeepSeek MTP engine families', () => { + const summary = buildOverviewModelSummary(Model.DeepSeek_V4_Pro, [ + ...frontier([1200, 1000, 800, 600], { framework: 'dynamo-trt' }), + ...frontier([1100, 900, 700, 500], { framework: 'mori-sglang' }), + ]); + + expect(summary.comparisonGroups.map(({ engineGroup }) => engineGroup).toSorted()).toEqual([ + 'sglang', + 'trt', + ]); + expect(summary.comparisonGroups.every((group) => group.primaryRanking.runnerUp === null)).toBe( + true, + ); + }); + + it('keeps aggregate and disaggregate denominators in separate cohorts', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { disagg: false }), + ...frontier([1800, 1500, 1200, 900], { + hardware: 'gb200', + disagg: true, + is_multinode: true, + }), + ]); + expect(summary.comparisonGroups.map(({ deploymentMode }) => deploymentMode).toSorted()).toEqual( + ['aggregated', 'disaggregated'], + ); + }); + + it('counts aggregated GPUs as one pool and disaggregated as prefill + decode', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { num_prefill_gpu: 4, num_decode_gpu: 4 }), + ...frontier([1800, 1500, 1200, 900], { + hardware: 'gb200', + disagg: true, + is_multinode: true, + num_prefill_gpu: 16, + num_decode_gpu: 16, + }), + ]); + const leaderFor = (mode: string) => + summary.comparisonGroups.find((group) => group.deploymentMode === mode)?.primaryRanking + .leader; + + expect(leaderFor('aggregated')?.totalGpu).toBe(4); + expect(leaderFor('disaggregated')?.totalGpu).toBe(32); + }); + + it('ranks 50 and 100 independently across hardware with a real gap', () => { + // b200 leads 50 (1000 vs 800) but bottoms out at 100 (300), where gb200's + // shallower curve (400) takes over — two exact reads, so the gap is real. + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 300], { hardware: 'b200' }), + ...frontier([1100, 800, 600, 400], { hardware: 'gb200' }), + ]); + const [group] = summary.comparisonGroups; + const expectedHighConfigKey = overviewConfigIdentityKey(row({ hardware: 'gb200' })); + + expect(group.primaryRanking.state).toBe('comparable'); + expect(group.primaryRanking.leader?.hardware).toBe('b200'); + expect(group.primaryRanking.runnerUp?.hardware).toBe('gb200'); + expect(group.primaryRanking.gapPercent).toBeCloseTo((1000 / 800 - 1) * 100); + expect(group.highRanking.leader?.key).toBe(expectedHighConfigKey); + expect(group.highRanking.leader?.hardware).toBe('gb200'); + expect(group.highLeaderTransition).toBe('changed_hardware'); + }); + + it('picks the best per-hardware config independently at each tier', () => { + // On b200, sglang wins 50 (1000 > 700) while dynamo-trt wins 100 (400 > 200). + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 200], { framework: 'sglang' }), + ...frontier([1100, 700, 500, 400], { framework: 'dynamo-trt' }), + ]); + const [group] = summary.comparisonGroups; + + expect(group.primaryRanking.leader).toMatchObject({ hardware: 'b200', framework: 'sglang' }); + expect(group.highRanking.leader).toMatchObject({ hardware: 'b200', framework: 'dynamo-trt' }); + expect(group.highLeaderTransition).toBe('same_hardware'); + }); + + it('reports no_primary_baseline when 50 has no exact read but 100 does', () => { + // Frontier floor at 60 tok/s/user: tier 50 is clamped_low (no exact read), + // so the primary tier has no leader, while tier 100 sits inside [60, 110] + // and interpolates — a high leader with no primary baseline to compare to. + // FP4 rows so this insufficient-coverage precision is the primary (0-0 exact + // tie → FP4); the transition mechanics under test are precision-agnostic. + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontierAt( + [ + [60, 900], + [75, 800], + [90, 700], + [110, 600], + ], + { precision: Precision.FP4 }, + ), + ]); + const [group] = summary.comparisonGroups; + + expect(group.primaryRanking.state).toBe('insufficient_coverage'); + expect(group.highLeaderTransition).toBe('no_primary_baseline'); + }); + + it('never lets clamped or unreachable reads lead or form a gap at a tier', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + // Frontier floor at 60 tok/s/user → 50 read is clamped_low. + ...frontierAt( + [ + [60, 900], + [70, 800], + [80, 700], + [90, 600], + ], + { hardware: 'b300', precision: Precision.FP4 }, + ), + // Frontier ceiling at 35 tok/s/user → 50 read is unreachable. + ...frontierAt( + [ + [20, 500], + [25, 450], + [30, 400], + [35, 350], + ], + { hardware: 'b200', precision: Precision.FP4 }, + ), + ]); + const [group] = summary.comparisonGroups; + + expect(group.primaryRanking.state).toBe('insufficient_coverage'); + expect(group.primaryRanking.gapPercent).toBeNull(); + expect(group.primaryRanking.leader).toBeNull(); + expect(group.hardwareStatuses.find((s) => s.hardware === 'b300')?.primary.boundary).toBe( + 'clamped_low', + ); + }); + + it('lets a lower exact read outrank a higher clamped one', () => { + // b300's clamped 900 is its frontier floor, not a 50 tok/s/user result, so + // gb200's real 500 leads and no gap is claimed against unmeasured coverage. + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontierAt( + [ + [60, 900], + [70, 800], + [80, 700], + [90, 600], + ], + { hardware: 'b300' }, + ), + ...frontier([700, 500, 400, 300], { hardware: 'gb200' }), + ]); + const [group] = summary.comparisonGroups; + + expect(group.primaryRanking.leader?.hardware).toBe('gb200'); + expect(group.primaryRanking.state).toBe('single_measured'); + expect(group.primaryRanking.gapPercent).toBeNull(); + expect(group.hardwareStatuses.find((s) => s.hardware === 'b300')?.primary).toMatchObject({ + value: 900, + boundary: 'clamped_low', + }); + }); + + it('dates a model by its newest workload row, and null when it has none', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { date: '2026-07-01' }), + ...frontier([1100, 900, 700, 500], { hardware: 'gb200', date: '2026-07-15' }), + ]); + + expect(summary.latestWorkloadDate).toBe('2026-07-15'); + expect(buildOverviewModelSummary(Model.Qwen3_5, []).latestWorkloadDate).toBeNull(); + }); +}); + +describe('overview precision selection and secondary', () => { + it('makes the wider exact-@50 coverage the primary precision', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { hardware: 'gb200', precision: Precision.FP8 }), + ...frontier([1000, 800, 600, 400], { hardware: 'gb300', precision: Precision.FP8 }), + ]); + + expect(summary.selectedPrecision).toBe(Precision.FP8); + }); + + it('breaks an exact-@50 coverage tie toward FP4', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { hardware: 'gb200', precision: Precision.FP8 }), + ]); + + expect(summary.selectedPrecision).toBe(Precision.FP4); + }); + + it('ranks each precision within itself, never across FP4 and FP8', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { hardware: 'b200', precision: Precision.FP4 }), + ...frontier([1000, 800, 600, 400], { hardware: 'gb200', precision: Precision.FP8 }), + ]); + + // The FP4 cohort ranks only its own two hardware; the FP8 platform is the + // secondary precision, never a cross-precision delta in the primary cohort. + expect(summary.comparisonGroups[0].hardwareStatuses.map((s) => s.hardware).toSorted()).toEqual([ + 'b200', + 'b300', + ]); + expect(summary.secondary?.precision).toBe(Precision.FP8); + }); + + it('keeps FP8 visible as secondary coverage when FP4 is primary', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { hardware: 'b200', precision: Precision.FP4 }), + ...frontier([1000, 800, 600, 400], { hardware: 'h200', precision: Precision.FP8 }), + ]); + + expect(summary.selectedPrecision).toBe(Precision.FP4); + expect(summary.secondary?.state).toBe('coverage'); + expect(summary.secondary?.measuredHardware).toContain(getHardwareConfig('h200').label); + }); + + it('opens secondary rows only when FP8 ranks a pair on hardware FP4 lacks', () => { + const ranked = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), + ...frontier([1150, 950, 750, 550], { hardware: 'b200', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { hardware: 'gb300', precision: Precision.FP4 }), + ...frontier([1000, 800, 600, 400], { hardware: 'gb200', precision: Precision.FP8 }), + ...frontier([980, 780, 580, 380], { hardware: 'mi355x', precision: Precision.FP8 }), + ]); + // FP4 leads on three hardware; FP8 ranks a comparable pair on two it lacks. + expect(ranked.selectedPrecision).toBe(Precision.FP4); + expect(ranked.secondary?.state).toBe('ranked'); + + const covered = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { hardware: 'gb200', precision: Precision.FP4 }), + ...frontier([1150, 950, 750, 550], { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontier([900, 850, 650, 450], { hardware: 'gb200', precision: Precision.FP8 }), + ...frontier([880, 820, 620, 420], { hardware: 'mi355x', precision: Precision.FP8 }), + ]); + // Same FP8 pair, but both hardware already rank in FP4 → no new coverage. + expect(covered.secondary?.state).toBe('coverage'); + }); +}); + +describe('overview not-ranked reasons', () => { + it('accounts for every page hardware with an exact value or one reason', () => { + const model = [ + ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), + ...frontier([1100, 900, 700, 500], { hardware: 'b200', precision: Precision.FP4 }), + // gb200 measured at the primary precision, but standard decode only. + ...frontier([1000, 800, 600, 400], { + hardware: 'gb200', + precision: Precision.FP4, + spec_method: 'none', + }), + ]; + // mi355x reaches the page order only because another model measured it. + const other = frontier([900, 800, 700, 600], { hardware: 'mi355x', precision: Precision.FP4 }); + const order = buildOverviewHardwareOrder([...model, ...other]); + const summary = buildOverviewModelSummary(Model.Qwen3_5, model, order); + + // b300/b200 rank (exact @50) and are absent; gb200 and mi355x each get one. + expect(summary.notRanked.map((entry) => [entry.hardware, entry.reason])).toEqual([ + ['gb200', 'standard_decode_only'], + ['mi355x', 'no_8k1k_data'], + ]); + }); + + it('separates a frontier that tops out below 50 from a missing @50 read', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + // b200 frontier ceiling at 45 tok/s/user → @50 unreachable. + ...frontierAt( + [ + [20, 500], + [30, 450], + [40, 400], + [45, 350], + ], + { hardware: 'b200', precision: Precision.FP4 }, + ), + // b300 frontier floor at 60 tok/s/user → @50 clamped_low, not unreachable. + ...frontierAt( + [ + [60, 900], + [70, 800], + [80, 700], + [90, 600], + ], + { hardware: 'b300', precision: Precision.FP4 }, + ), + ]); + const reasonOf = (hardware: string) => + summary.notRanked.find((entry) => entry.hardware === hardware)?.reason; + + expect(reasonOf('b200')).toBe('cannot_reach_at50'); + expect(reasonOf('b300')).toBe('no_exact_at50'); + }); +}); + +describe('overview result-level evidence dates', () => { + it('dates each read from its own config frontier, never a sibling', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...datedFrontier( + [ + [40, 1100, '2026-07-10'], + [60, 900, '2026-07-10'], + ], + { hardware: 'b300', precision: Precision.FP4 }, + ), + ...datedFrontier( + [ + [40, 900, '2026-07-16'], + [60, 700, '2026-07-16'], + ], + { hardware: 'b200', precision: Precision.FP4 }, + ), + ]); + + expect(statusOf(summary, 'b300')?.primary.evidenceDate).toEqual({ + from: '2026-07-10', + to: '2026-07-10', + }); + expect(statusOf(summary, 'b200')?.primary.evidenceDate).toEqual({ + from: '2026-07-16', + to: '2026-07-16', + }); + }); + + it('collapses a same-day bracket and spans a cross-day one', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...datedFrontier( + [ + [40, 1100, '2026-07-10'], + [60, 900, '2026-07-10'], + ], + { hardware: 'b300', precision: Precision.FP4 }, + ), + ...datedFrontier( + [ + [40, 900, '2026-06-24'], + [60, 700, '2026-07-04'], + ], + { hardware: 'b200', precision: Precision.FP4 }, + ), + ]); + + expect(statusOf(summary, 'b300')?.primary.evidenceDate).toEqual({ + from: '2026-07-10', + to: '2026-07-10', + }); + expect(statusOf(summary, 'b200')?.primary.evidenceDate).toEqual({ + from: '2026-06-24', + to: '2026-07-04', + }); + }); + + it('ignores a hidden slower config even when it is newer', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...datedFrontier( + [ + [40, 1100, '2026-07-10'], + [60, 900, '2026-07-10'], + ], + { hardware: 'b300', precision: Precision.FP4, framework: 'sglang' }, + ), + // Slower same-hardware config, newer runs — never backs the visible read. + ...datedFrontier( + [ + [40, 700, '2026-07-20'], + [60, 500, '2026-07-20'], + ], + { hardware: 'b300', precision: Precision.FP4, framework: 'dynamo-trt' }, + ), + ]); + + expect(statusOf(summary, 'b300')?.primary.evidenceDate).toEqual({ + from: '2026-07-10', + to: '2026-07-10', + }); + }); + + it('dates @50 and @100 from their own bracketing knots', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...datedFrontier( + [ + [40, 1200, '2026-07-01'], + [60, 1000, '2026-07-01'], + [90, 700, '2026-07-15'], + [110, 500, '2026-07-15'], + ], + { hardware: 'b300', precision: Precision.FP4 }, + ), + ]); + const status = statusOf(summary, 'b300'); + + expect(status?.primary.evidenceDate).toEqual({ from: '2026-07-01', to: '2026-07-01' }); + expect(status?.high.evidenceDate).toEqual({ from: '2026-07-15', to: '2026-07-15' }); + }); +}); + +describe('overview hardware coverage', () => { + it('holds hardware order fixed against input order and performance changes', () => { + const expected = ['gb300', 'gb200', 'b300', 'b200', 'mi355x']; + const build = (throughputs: number[], hardware: string[]) => { + const rows = hardware.flatMap((hw, index) => + frontier( + [ + throughputs[index] + 200, + throughputs[index], + throughputs[index] - 200, + throughputs[index] - 400, + ], + { hardware: hw }, + ), + ); + return buildOverviewModelSummary(Model.Qwen3_5, rows, buildOverviewHardwareOrder(rows)); + }; + + // Same throughputs, hardware fed in the opposite order: the fastest platform + // swaps from mi355x to gb300 while the display order must not move. + const mi355xLeads = build([600, 700, 800, 900, 1000], expected); + const gb300Leads = build([600, 700, 800, 900, 1000], [...expected].toReversed()); + + expect(mi355xLeads.hardwareStatuses.map(({ hardware }) => hardware)).toEqual(expected); + expect(gb300Leads.hardwareStatuses.map(({ hardware }) => hardware)).toEqual(expected); + expect(primaryLeaderOf(mi355xLeads)).toMatchObject({ hardware: 'mi355x' }); + expect(primaryLeaderOf(gb300Leads)).toMatchObject({ hardware: 'gb300' }); + expect(gb300Leads.comparisonGroups[0].hardwareStatuses.map(({ hardware }) => hardware)).toEqual( + expected, + ); + }); + + it('separates unsupported-precision coverage from hardware this model never ran', () => { + // Kimi shape: FP4 standard decode on B200, INT4 only on H200. H200 and + // MI355X reach the page order only because another model measured them. + const kimi = [ + ...frontier([1200, 1000, 800, 600], { + model: 'kimik2.5', + hardware: 'b200', + precision: Precision.FP4, + spec_method: 'none', + }), + ...frontier([900, 800, 700, 600], { + model: 'kimik2.5', + hardware: 'h200', + precision: Precision.INT4, + spec_method: 'none', + }), + ]; + const otherModel = [ + ...frontier([900, 800, 700, 600], { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontier([700, 600, 500, 400], { hardware: 'h200', precision: Precision.FP8 }), + ]; + const order = buildOverviewHardwareOrder([...kimi, ...otherModel]); + const summary = buildOverviewModelSummary(Model.Kimi_K2_5, kimi, order); + + expect(order.map(({ hardware }) => hardware)).toEqual(['b200', 'mi355x', 'h200']); + expect(summary.hardwareStatuses.map(({ hardware }) => hardware)).toEqual([ + 'b200', + 'mi355x', + 'h200', + ]); + expect(statusOf(summary, 'b200')?.coverage.kind).toBe('standard_only'); + expect(statusOf(summary, 'h200')?.coverage).toMatchObject({ + kind: 'unsupported_precision_only', + availablePrecisions: [Precision.INT4], + }); + expect(statusOf(summary, 'mi355x')?.coverage).toMatchObject({ + kind: 'no_workload_data', + availablePrecisions: [], + }); + }); +}); + +// Drift guard for the synthetic overview e2e fixture: if the assembler contract +// changes, the hand-built rows in overview-rows.json must be regenerated, and +// this fails loudly instead of the e2e stranding empty data. Every expectation +// is derived by running this same assembler over the fixture — never eyeballed — +// so it locks the exact 8.7 states overview.cy.ts renders against. +describe('assembleOverviewPageData over the overview-rows fixture', () => { + it('serves every 8.7 precision, coverage and evidence-date state through the real builder', () => { + const page = assembleOverviewPageData( + overviewRowsFixture as unknown as Record, + ); + + expect(page.models).toHaveLength(DEFAULT_MODELS.size); + expect(page.datasetThroughDate).toBe('2026-07-18'); + expect(page.hardwareOrder.map((entry) => entry.hardware)).toEqual([ + 'gb300', + 'gb200', + 'b300', + 'b200', + 'mi355x', + 'h200', + 'mi325x', + 'h100', + ]); + + // DeepSeek: FP4 primary ranks B300 over B200; FP8 stays visible as one + // coverage line (single measured hardware, not a second table); every other + // page hardware carries exactly one not-ranked reason, including BOTH clamp + // directions; and the leader's @50 read spans two calendar days — the only + // cross-day evidence range in the fixture, so the e2e can exercise the range + // label the current live dataset never produces. + const deepseek = page.models.find((m) => m.model === Model.DeepSeek_V4_Pro); + expect(deepseek?.selectedPrecision).toBe(Precision.FP4); + const dsCohort = deepseek?.comparisonGroups[0]; + expect(dsCohort?.primaryRanking.state).toBe('comparable'); + expect(dsCohort?.primaryRanking.leader?.hardware).toBe('b300'); + expect(dsCohort?.highRanking.leader?.hardware).toBe('b200'); + expect( + dsCohort?.hardwareStatuses.find((s) => s.hardware === 'b300')?.primary.evidenceDate, + ).toEqual({ from: '2026-06-24', to: '2026-07-04' }); + expect(deepseek?.secondary?.state).toBe('coverage'); + expect(deepseek?.secondary?.precision).toBe(Precision.FP8); + expect(deepseek?.secondary?.measuredHardware).toEqual(['GB200 NVL72']); + expect( + Object.fromEntries( + (deepseek?.notRanked ?? []).map((entry) => [entry.hardware, entry.reason]), + ), + ).toMatchObject({ + gb300: 'cannot_reach_at50', + mi355x: 'no_exact_at50', + h100: 'standard_decode_only', + gb200: 'other_precision_only', + h200: 'no_8k1k_data', + }); + + // MiniMax: the wider exact-@50 FP8 coverage flips the primary precision to + // FP8 (the coverage rule), dropping FP4 to a single-hardware coverage line. + const minimax = page.models.find((m) => m.model === Model.MiniMax_M3); + expect(minimax?.selectedPrecision).toBe(Precision.FP8); + expect(minimax?.comparisonGroups[0]?.primaryRanking.leader?.hardware).toBe('h200'); + expect(minimax?.secondary?.state).toBe('coverage'); + expect(minimax?.secondary?.precision).toBe(Precision.FP4); + expect(minimax?.secondary?.measuredHardware).toEqual(['H200']); + + // Qwen: FP4 primary, and FP8 secondary earns a FULL ranked block — it ranks a + // comparable pair AND adds MI355X, hardware FP4 has no exact @50 read for. + const qwen = page.models.find((m) => m.model === Model.Qwen3_5); + expect(qwen?.selectedPrecision).toBe(Precision.FP4); + expect(qwen?.secondary?.state).toBe('ranked'); + expect(qwen?.secondary?.precision).toBe(Precision.FP8); + const qwenSecondaryCohort = qwen?.secondary?.comparisonGroups[0]; + expect(qwenSecondaryCohort?.primaryRanking.state).toBe('comparable'); + expect(qwenSecondaryCohort?.primaryRanking.leader?.hardware).toBe('b200'); + expect(qwen?.secondary?.measuredHardware).toEqual(['B200', 'MI355X']); + }); +}); diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts new file mode 100644 index 00000000..6a4e4d59 --- /dev/null +++ b/packages/app/src/lib/overview-data.ts @@ -0,0 +1,969 @@ +import { resolveFrameworkPartLabel } from '@semianalysisai/inferencex-constants'; + +import { parallelismLabel } from '@/components/inference/utils/parallelism-label'; + +import type { BenchmarkRow } from './api'; +import { buildAvailabilityHwKey } from './chart-utils'; +import { getHardwareConfig, getModelSortIndex } from './constants'; +import { + DEFAULT_MODELS, + getModelExclusion, + getModelLabel, + Precision, + type Model, +} from './data-mappings'; +import { buildExclusion, type Exclusion } from './exclusion'; +import { overviewConfigIdentityKey } from './overview-config-identity'; +import { computeTcoFeed, type TcoTierBoundary } from './tco-feed'; + +export const OVERVIEW_WORKLOAD = { isl: 8192, osl: 1024 } as const; +export const OVERVIEW_TIERS = [30, 50, 75, 100] as const; +/** Headline service point every platform is ranked at. */ +export const OVERVIEW_PRIMARY_TIER = 50; +/** High-interactivity capability read; also drives the high-tier leader transition. */ +export const OVERVIEW_HIGH_TIER = 100; + +export interface OverviewTierValue { + tier: number; + value: number | null; + boundary: TcoTierBoundary; + /** + * Run dates of the frontier point(s) backing this value — the two bracketing + * points when interpolated (from = earlier, to = later; equal on the same + * day), the single point twice when clamped, null when `value` is null. Comes + * only from this config's own frontier at this tier, never a sibling or cohort. + */ + evidenceDate: { from: string; to: string } | null; +} + +/** + * One real deployable serving configuration, identified by its exact + * deployment topology (model × hardware × framework × precision × spec_method × + * disagg × multinode × per-role parallelism × GPU counts × offload). Tier + * values come from this configuration's own Pareto frontier only — never + * blended across configurations. + */ +export interface OverviewConfigResult { + key: string; + dbModel: string; + hardware: string; + hardwareLabel: string; + hwKey: string; + framework: string; + frameworkLabel: string; + specMethod: string; + specLabel: string; + disagg: boolean; + precision: string; + offloadMode: string; + isMultinode: boolean; + numPrefillGpu: number; + numDecodeGpu: number; + /** Physical GPUs backing this config: one shared pool when aggregated, prefill + * plus decode when disaggregated. */ + totalGpu: number; + parallelism: string; + image: string | null; + sourceRunUrls: string[]; + tierValues: OverviewTierValue[]; + latestDate: string; + oldestFrontierDate: string; +} + +/** One hardware's frontier read at a single tier, with its backing config. */ +export interface OverviewTierRead { + tier: number; + value: number | null; + boundary: TcoTierBoundary | null; + /** Copied from the backing config's tier value; null when there is no read. */ + evidenceDate: { from: string; to: string } | null; + config: OverviewConfigResult | null; +} + +/** + * Why a hardware is or is not competing in the model's ranking. Coverage is + * derived from the raw workload rows alone, so a hardware never disappears just + * because a filter or a ranking excluded it: + * + * - `comparable_spec` — speculative results at the selected precision, ranked; + * - `standard_only` — selected precision measured, but standard decode only; + * - `alternate_precision_only` — measured at the other FP4/FP8 precision; + * - `other_engine_group_only` — speculative, but only in an engine + * comparability group no other hardware here shares; + * - `unsupported_precision_only` — measured only outside FP4/FP8; + * - `no_workload_data` — no 8K/1K single-turn row for this model. + */ +export type OverviewCoverageKind = + | 'comparable_spec' + | 'standard_only' + | 'alternate_precision_only' + | 'other_engine_group_only' + | 'unsupported_precision_only' + | 'no_workload_data'; + +export interface OverviewHardwareCoverage { + hardware: string; + hardwareLabel: string; + kind: OverviewCoverageKind; + availablePrecisions: string[]; +} + +/** One hardware in the page-wide display order, which never encodes performance. */ +export interface OverviewHardwareOrderEntry { + hardware: string; + hardwareLabel: string; +} + +export interface OverviewHardwareStatus { + hardware: string; + hardwareLabel: string; + coverage: OverviewHardwareCoverage; + /** + * Primary-tier read of this hardware's best configuration in the surrounding + * config set — the same read the ranking used, so a row can never display a + * number the ranking did not. Null read when the hardware has no ranked + * configuration; `coverage.kind` says why. + */ + primary: OverviewTierRead; + /** High-tier read, taken from this hardware's own best config at that tier. */ + high: OverviewTierRead; + isPrimaryLeader: boolean; + /** Signed percentage against the primary leader; null for the leader itself + * and whenever either side is not an exact in-range read. */ + primaryDeltaPercent: number | null; +} + +/** Whether a cohort's throughput denominator is aggregated or disaggregated. */ +export type OverviewDeploymentMode = 'aggregated' | 'disaggregated'; + +/** + * Ranking state at one tier: `comparable` (≥2 exact hardware reads, gap + * computable), `single_measured` (one exact read), or `insufficient_coverage` + * (no exact read — every read is clamped or unreachable). + */ +export type OverviewRankingState = 'comparable' | 'single_measured' | 'insufficient_coverage'; + +export interface OverviewTierRanking { + tier: number; + state: OverviewRankingState; + leader: OverviewConfigResult | null; + runnerUp: OverviewConfigResult | null; + gapPercent: number | null; +} + +/** + * One comparable cohort: exact configs that share an engine comparability group + * (per the model's exclusion policy) and a deployment mode. Tiers are ranked + * independently inside a cohort; configs never compare across cohorts. + */ +export interface OverviewComparisonGroup { + id: string; + dbModel: string; + engineGroup: string | null; + deploymentMode: OverviewDeploymentMode; + hardwareStatuses: OverviewHardwareStatus[]; + primaryRanking: OverviewTierRanking; + highRanking: OverviewTierRanking; + /** + * How the high-tier leader relates to the primary-tier leader: a new hardware + * took the lead (`changed_hardware`), the same hardware held it + * (`same_hardware`), there was no primary leader to compare against + * (`no_primary_baseline`), or the high tier has no leader at all (`null`). + */ + highLeaderTransition: 'same_hardware' | 'changed_hardware' | 'no_primary_baseline' | null; +} + +export interface OverviewSecondaryPrecision { + precision: string; + /** 'ranked' → full rows render; 'coverage' → one compact line renders. */ + state: 'ranked' | 'coverage'; + /** Populated when state === 'ranked'; [] otherwise. Built by the SAME pipeline as the primary (cohorts, rankings, transitions) over the secondary precision's rows. */ + comparisonGroups: OverviewComparisonGroup[]; + hardwareStatuses: OverviewHardwareStatus[]; + /** Hardware labels measured at this precision (any decode mode), page order. Always populated. */ + measuredHardware: string[]; +} + +/** + * Why a page hardware carries no visible value for a model. Derived from the + * existing coverage kinds and ranking outcomes, never a parallel truth: + * + * - `standard_decode_only` — from `standard_only`; + * - `other_precision_only` — from `alternate_precision_only` and unranked by + * the secondary block; the renderer prints 'FP8 only'/'FP4 only'; + * - `int4_bf16_only` — from `unsupported_precision_only`; + * - `different_serving_cohort` — from `other_engine_group_only`; + * - `no_8k1k_data` — from `no_workload_data`; + * - `cannot_reach_at50` — comparable spec whose frontier tops out below 50; + * - `no_exact_at50` — comparable spec with no exact @50 read for any other reason. + */ +export type OverviewNotRankedReason = + | 'standard_decode_only' + | 'other_precision_only' + | 'int4_bf16_only' + | 'different_serving_cohort' + | 'no_8k1k_data' + | 'cannot_reach_at50' + | 'no_exact_at50'; + +export interface OverviewNotRankedEntry { + hardware: string; + hardwareLabel: string; + reason: OverviewNotRankedReason; + /** FP4/FP8 precisions this hardware measured, for the 'FP8 only' style copy. */ + precisions: string[]; +} + +export interface OverviewModelSummary { + model: Model; + modelLabel: string; + /** + * Primary precision: whichever of FP4/FP8 covers more unique hardware with an + * exact @50 speculative read; a tie (including 0-0 when FP4/FP8 rows exist) + * goes to FP4. Null only when the model has no FP4/FP8 workload rows at all. + */ + selectedPrecision: string | null; + /** + * Every page hardware's coverage and tier reads for this model, in the + * page-wide order. Leadership is claimed here only when a single cohort + * exists; with several cohorts the claim belongs to `comparisonGroups`. + */ + hardwareStatuses: OverviewHardwareStatus[]; + /** Comparable cohorts, each ranked independently at the primary and high tiers. */ + comparisonGroups: OverviewComparisonGroup[]; + /** The other of FP4/FP8, or null when it has no workload rows. Ranks only + * within itself — never against the primary precision. */ + secondary: OverviewSecondaryPrecision | null; + /** + * One entry per page hardware with no visible value — neither an exact @50 + * read in the primary block nor a ranked read in a `ranked` secondary. Every + * page hardware is therefore accounted for per model: an exact value, or one + * not-ranked reason. + */ + notRanked: OverviewNotRankedEntry[]; + /** + * Newest workload row measured for this model. Now that every visible result + * carries its own `evidenceDate`, this exists ONLY to date a coverage-only + * model with zero visible results. Null when the model has no workload row. + */ + latestWorkloadDate: string | null; + emptyReason: 'no_8k1k_single_turn_data' | 'no_fp4_fp8_data' | null; +} + +export interface OverviewPageData { + models: OverviewModelSummary[]; + hardwareOrder: { hardware: string; hardwareLabel: string }[]; + datasetThroughDate: string | null; +} + +/** Precisions the overview may rank, in preference order. */ +const OVERVIEW_PRECISIONS: readonly string[] = [Precision.FP4, Precision.FP8]; + +/** Rows measuring the fixed overview workload, before any other filter. */ +function overviewWorkloadRows(rows: readonly BenchmarkRow[]): BenchmarkRow[] { + return rows.filter( + (row) => + row.benchmark_type === 'single_turn' && + row.isl === OVERVIEW_WORKLOAD.isl && + row.osl === OVERVIEW_WORKLOAD.osl, + ); +} + +/** + * Page-wide hardware display order: every base hardware with FP4/FP8 coverage + * at the overview workload, in hardware-registry order. Built before precision + * selection and before any ranking, so neither a precision fallback nor a + * measurement change can reorder or drop a column. + */ +export function buildOverviewHardwareOrder( + rows: readonly BenchmarkRow[], +): OverviewHardwareOrderEntry[] { + const hardware = new Set( + overviewWorkloadRows(rows) + .filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)) + .map((row) => row.hardware), + ); + return [...hardware] + .toSorted((a, b) => getModelSortIndex(a) - getModelSortIndex(b) || a.localeCompare(b)) + .map((entry) => ({ hardware: entry, hardwareLabel: getHardwareConfig(entry).label })); +} + +/** + * Newest raw workload row across the page. Deliberately not derived from the + * retained winners: a precision or engine the page never ranks still dates the + * dataset it was measured in. + */ +export function overviewDatasetThroughDate(rows: readonly BenchmarkRow[]): string | null { + return overviewWorkloadRows(rows).reduce( + (latest, row) => (latest === null || row.date > latest ? row.date : latest), + null, + ); +} + +/** Newest date among a model's own workload rows; null when it has none. */ +function latestWorkloadDateOf(workloadRows: readonly BenchmarkRow[]): string | null { + return workloadRows.reduce( + (latest, row) => (latest === null || row.date > latest ? row.date : latest), + null, + ); +} + +/** + * Speculative configs for one precision, grouped by exact deployment identity. + * Standard-decode rows never enter the pool, so a precision is ranked on its + * speculative frontier alone. + */ +function buildPrecisionConfigs( + model: Model, + workloadRows: readonly BenchmarkRow[], + precision: string, +): OverviewConfigResult[] { + const rowsByConfig = new Map(); + for (const row of workloadRows) { + if (row.precision !== precision || row.spec_method === 'none') continue; + const key = overviewConfigIdentityKey(row); + const configRows = rowsByConfig.get(key); + if (configRows) configRows.push(row); + else rowsByConfig.set(key, [row]); + } + + const configs: OverviewConfigResult[] = []; + for (const [key, configRows] of rowsByConfig) { + const config = buildConfigResult(model, precision, key, configRows); + if (config) configs.push(config); + } + return configs; +} + +/** Unique hardware whose best @50 read is exact (in-range) among `configs`. */ +function exactPrimaryHardware(configs: readonly OverviewConfigResult[]): Set { + return new Set( + [...readsByHardwareAtTier(configs, OVERVIEW_PRIMARY_TIER).values()] + .filter(isExactTierRead) + .map((read) => read.config.hardware), + ); +} + +/** Whichever precision covers more exact-@50 hardware; a tie (incl. 0-0) → FP4. */ +function selectPrimaryFrom( + configsByPrecision: ReadonlyMap, +): string { + const exactCount = (precision: string) => + exactPrimaryHardware(configsByPrecision.get(precision) ?? []).size; + return exactCount(Precision.FP8) > exactCount(Precision.FP4) ? Precision.FP8 : Precision.FP4; +} + +/** + * Primary precision by exact-@50 hardware coverage: FP4 and FP8 are built and + * counted independently, the wider coverage wins, and a tie (including 0-0 when + * FP4/FP8 rows exist) goes to FP4. Null only when no FP4/FP8 workload row exists + * — coverage volume of one vendor's curves can shift which precision is primary, + * but never which precisions are ranked. + */ +export function selectOverviewPrecision( + model: Model, + workloadRows: readonly BenchmarkRow[], +): string | null { + if (!workloadRows.some((row) => OVERVIEW_PRECISIONS.includes(row.precision))) return null; + const configsByPrecision = new Map( + OVERVIEW_PRECISIONS.map( + (precision) => [precision, buildPrecisionConfigs(model, workloadRows, precision)] as const, + ), + ); + return selectPrimaryFrom(configsByPrecision); +} + +function tierValueAt(config: OverviewConfigResult, tier: number): number | null { + return config.tierValues.find((value) => value.tier === tier)?.value ?? null; +} + +export function overviewPrimaryValue(config: OverviewConfigResult): number | null { + return tierValueAt(config, OVERVIEW_PRIMARY_TIER); +} + +export function overviewHighValue(config: OverviewConfigResult): number | null { + return tierValueAt(config, OVERVIEW_HIGH_TIER); +} + +function readConfigAtTier(config: OverviewConfigResult, tier: number): OverviewTierRead { + const tierValue = config.tierValues.find((value) => value.tier === tier); + return { + tier, + value: tierValue?.value ?? null, + boundary: tierValue?.boundary ?? null, + evidenceDate: tierValue?.evidenceDate ?? null, + config, + }; +} + +/** A tier read known to be backed by a configuration. */ +interface ConfigTierRead extends OverviewTierRead { + config: OverviewConfigResult; +} + +/** + * A tier read counts only when it lands inside the configuration's own measured + * frontier. A clamped or unreachable read is a coverage gap, so it can never + * lead a tier or anchor a percentage gap. + */ +const isExactTierRead = (read: T): read is T & { value: number } => + read.value !== null && read.boundary === 'interpolated'; + +function compareTierReads(a: ConfigTierRead, b: ConfigTierRead): number { + return ( + (b.value ?? -1) - (a.value ?? -1) || + getModelSortIndex(a.config.hardware) - getModelSortIndex(b.config.hardware) || + a.config.key.localeCompare(b.config.key) + ); +} + +/** + * One read per hardware at `tier`: its best exact in-range read, or — only when + * the hardware has no exact read at all — its best out-of-range read, kept so + * the hardware still surfaces as a coverage gap. Ranking and hardware rows both + * consume this, so a row can never display a number the ranking did not use. + */ +function readsByHardwareAtTier( + configs: readonly OverviewConfigResult[], + tier: number, +): Map { + const reads = configs + .map((config): ConfigTierRead => ({ ...readConfigAtTier(config, tier), config })) + .toSorted(compareTierReads); + + const byHardware = new Map(); + for (const read of reads.filter(isExactTierRead)) { + if (!byHardware.has(read.config.hardware)) byHardware.set(read.config.hardware, read); + } + for (const read of reads) { + if (!byHardware.has(read.config.hardware)) byHardware.set(read.config.hardware, read); + } + return byHardware; +} + +function nullTierRead(tier: number): OverviewTierRead { + return { tier, value: null, boundary: null, evidenceDate: null, config: null }; +} + +/** + * Classify one hardware's coverage from its raw workload rows. `comparable` + * lists the hardware sharing an engine comparability group with at least one + * other hardware; a platform whose only speculative results sit in an engine + * family nobody else here uses is reported as coverage rather than a competitor. + */ +function coverageKindFor( + hardwareRows: readonly BenchmarkRow[], + selectedPrecision: string | null, + hardware: string, + comparable: ReadonlySet, +): OverviewCoverageKind { + if (hardwareRows.length === 0) return 'no_workload_data'; + const candidates = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); + if (candidates.length === 0) return 'unsupported_precision_only'; + const selected = candidates.filter((row) => row.precision === selectedPrecision); + if (selected.length === 0) return 'alternate_precision_only'; + if (!selected.some((row) => row.spec_method !== 'none')) return 'standard_only'; + return comparable.size === 0 || comparable.has(hardware) + ? 'comparable_spec' + : 'other_engine_group_only'; +} + +/** + * One status row per entry of `coverage`, which already carries the stable + * hardware order. `configs` scopes the tier reads and the leader: the whole + * model's ranked set at model level, one cohort's set inside a comparison group. + */ +function buildHardwareStatuses( + coverage: readonly OverviewHardwareCoverage[], + configs: readonly OverviewConfigResult[], + withLeader: boolean, +): OverviewHardwareStatus[] { + const primaryReads = readsByHardwareAtTier(configs, OVERVIEW_PRIMARY_TIER); + const highReads = readsByHardwareAtTier(configs, OVERVIEW_HIGH_TIER); + const leader = withLeader + ? [...primaryReads.values()].filter(isExactTierRead).toSorted(compareTierReads)[0] + : undefined; + + return coverage.map((hardwareCoverage) => { + const { hardware, hardwareLabel } = hardwareCoverage; + const primary = primaryReads.get(hardware) ?? nullTierRead(OVERVIEW_PRIMARY_TIER); + const isPrimaryLeader = leader !== undefined && primary.config?.key === leader.config.key; + return { + hardware, + hardwareLabel, + coverage: hardwareCoverage, + primary, + high: highReads.get(hardware) ?? nullTierRead(OVERVIEW_HIGH_TIER), + isPrimaryLeader, + primaryDeltaPercent: + leader !== undefined && !isPrimaryLeader && isExactTierRead(primary) && leader.value > 0 + ? (primary.value / leader.value - 1) * 100 + : null, + }; + }); +} + +/** + * Rank one comparable cohort at a single tier. Every configuration is read at + * `tier`, each hardware contributes only its best exact read, and the surviving + * hardware are ordered by that read — so each tier is ranked on its own + * evidence and a 50 winner is never carried into the 100 ranking. + */ +function rankAtTier(configs: readonly OverviewConfigResult[], tier: number): OverviewTierRanking { + const hardwareLeaders = [...readsByHardwareAtTier(configs, tier).values()] + .filter(isExactTierRead) + .toSorted(compareTierReads); + + const [leader, runnerUp] = hardwareLeaders; + return { + tier, + state: + hardwareLeaders.length >= 2 + ? 'comparable' + : hardwareLeaders.length === 1 + ? 'single_measured' + : 'insufficient_coverage', + leader: leader?.config ?? null, + runnerUp: runnerUp?.config ?? null, + gapPercent: + leader && runnerUp && runnerUp.value > 0 ? (leader.value / runnerUp.value - 1) * 100 : null, + }; +} + +interface OverviewCohort { + dbModel: string; + engineGroup: string | null; + deploymentMode: OverviewDeploymentMode; + configs: OverviewConfigResult[]; +} + +/** + * How a cohort's high-tier leader relates to its primary-tier leader. Compared + * on hardware, not config identity: a same-hardware engine or precision swap + * between tiers is not a hardware change, and a high tier with no primary + * baseline to compare against says so rather than implying a change. + */ +function computeHighLeaderTransition( + primaryRanking: OverviewTierRanking, + highRanking: OverviewTierRanking, +): OverviewComparisonGroup['highLeaderTransition'] { + if (highRanking.leader === null) return null; + if (primaryRanking.leader === null) return 'no_primary_baseline'; + return highRanking.leader.hardware === primaryRanking.leader.hardware + ? 'same_hardware' + : 'changed_hardware'; +} + +function buildComparisonGroup( + id: string, + cohort: OverviewCohort, + coverage: readonly OverviewHardwareCoverage[], +): OverviewComparisonGroup { + const cohortHardware = new Set(cohort.configs.map(({ hardware }) => hardware)); + const primaryRanking = rankAtTier(cohort.configs, OVERVIEW_PRIMARY_TIER); + const highRanking = rankAtTier(cohort.configs, OVERVIEW_HIGH_TIER); + return { + id, + dbModel: cohort.dbModel, + engineGroup: cohort.engineGroup, + deploymentMode: cohort.deploymentMode, + hardwareStatuses: buildHardwareStatuses( + coverage.filter(({ hardware }) => cohortHardware.has(hardware)), + cohort.configs, + true, + ), + primaryRanking, + highRanking, + highLeaderTransition: computeHighLeaderTransition(primaryRanking, highRanking), + }; +} + +/** + * Partition configurations into cohorts that may actually be compared: only + * configs sharing a db model (one display model spans several point releases, + * which must never rank against each other), an engine comparability group (the + * model's exclusion policy — DeepSeek MTP acceptance forcing differs per engine + * family) and a throughput denominator (aggregated vs disaggregated) are ranked + * against each other. + */ +function buildCohorts( + exclusion: Exclusion, + configs: readonly OverviewConfigResult[], +): Map { + const cohorts = new Map(); + for (const config of configs) { + const engineGroup = exclusion.groupOf(config.hwKey); + const deploymentMode: OverviewDeploymentMode = config.disagg ? 'disaggregated' : 'aggregated'; + const id = `${config.dbModel}|${engineGroup ?? 'any'}|${deploymentMode}`; + const cohort = cohorts.get(id); + if (cohort) cohort.configs.push(config); + else + cohorts.set(id, { dbModel: config.dbModel, engineGroup, deploymentMode, configs: [config] }); + } + return cohorts; +} + +/** + * Hardware sharing an engine comparability group with at least one other + * platform. Partitioned by engine group alone, never by the ranking cohorts: + * cohorts also split on deployment mode, so a platform measured only + * disaggregated would look isolated while running the very engine family its + * peers run — and `other_engine_group_only` would then assert a reason that is + * false. + */ +function comparableHardware( + exclusion: Exclusion, + configs: readonly OverviewConfigResult[], +): Set { + const byEngineGroup = new Map>(); + for (const config of configs) { + const group = exclusion.groupOf(config.hwKey); + const groupHardware = byEngineGroup.get(group); + if (groupHardware) groupHardware.add(config.hardware); + else byEngineGroup.set(group, new Set([config.hardware])); + } + return new Set( + [...byEngineGroup.values()] + .filter((groupHardware) => groupHardware.size > 1) + .flatMap((groupHardware) => [...groupHardware]), + ); +} + +function buildComparisonGroups( + cohorts: ReadonlyMap, + coverage: readonly OverviewHardwareCoverage[], +): OverviewComparisonGroup[] { + const leadValue = (group: OverviewComparisonGroup): number => + group.primaryRanking.leader ? (overviewPrimaryValue(group.primaryRanking.leader) ?? -1) : -1; + return [...cohorts.entries()] + .map(([id, cohort]) => buildComparisonGroup(id, cohort, coverage)) + .toSorted((a, b) => leadValue(b) - leadValue(a) || a.id.localeCompare(b.id)); +} + +function buildConfigResult( + model: Model, + precision: string, + key: string, + rows: BenchmarkRow[], +): OverviewConfigResult | null { + const feed = computeTcoFeed(rows, [OVERVIEW_WORKLOAD], OVERVIEW_TIERS); + if (feed.length === 0) return null; + + const first = rows[0]; + const { hardware, framework, spec_method: specMethod, disagg } = first; + const latestRow = rows.reduce((a, b) => (b.date > a.date ? b : a)); + const sourceRunUrls = [ + ...new Set(rows.flatMap((row) => (row.run_url === null ? [] : [row.run_url]))), + ].toSorted(); + return { + key, + dbModel: first.model, + hardware, + hardwareLabel: getHardwareConfig(hardware, model).label, + hwKey: buildAvailabilityHwKey(hardware, framework, specMethod, disagg), + framework, + frameworkLabel: resolveFrameworkPartLabel(model, framework), + specMethod, + specLabel: resolveFrameworkPartLabel(model, specMethod), + disagg, + precision, + offloadMode: first.offload_mode ?? 'off', + isMultinode: first.is_multinode, + numPrefillGpu: first.num_prefill_gpu, + numDecodeGpu: first.num_decode_gpu, + totalGpu: disagg ? first.num_prefill_gpu + first.num_decode_gpu : first.num_decode_gpu, + parallelism: parallelismLabel({ + tp: first.decode_tp, + ep: first.decode_ep, + dpAttention: first.decode_dp_attention, + disagg: first.disagg, + isMultinode: first.is_multinode, + prefillTp: first.prefill_tp, + prefillEp: first.prefill_ep, + prefillDpAttention: first.prefill_dp_attention, + prefillNumWorkers: first.prefill_num_workers, + decodeTp: first.decode_tp, + decodeEp: first.decode_ep, + decodeDpAttention: first.decode_dp_attention, + decodeNumWorkers: first.decode_num_workers, + }), + image: latestRow.image, + sourceRunUrls, + tierValues: feed.map((row) => { + const value = + row.boundary === 'unreachable' && row.output_tput_per_gpu === 0 + ? null + : row.output_tput_per_gpu; + return { + tier: row.tier, + value, + boundary: row.boundary, + evidenceDate: value === null ? null : row.evidence_date, + }; + }), + latestDate: feed[0].latest_date, + oldestFrontierDate: feed[0].oldest_frontier_date, + }; +} + +/** + * Coverage for every hardware in `order`, derived from raw workload rows only. + * Hardware the model never measured stays in the list as `no_workload_data`, + * so a platform is never silently absent from one model's row. + */ +function buildCoverage( + order: readonly OverviewHardwareOrderEntry[], + workloadRows: readonly BenchmarkRow[], + selectedPrecision: string | null, + comparable: ReadonlySet, +): OverviewHardwareCoverage[] { + const rowsByHardware = new Map(); + for (const row of workloadRows) { + const hardwareRows = rowsByHardware.get(row.hardware); + if (hardwareRows) hardwareRows.push(row); + else rowsByHardware.set(row.hardware, [row]); + } + + return order.map(({ hardware, hardwareLabel }) => { + const hardwareRows = rowsByHardware.get(hardware) ?? []; + return { + hardware, + hardwareLabel, + kind: coverageKindFor(hardwareRows, selectedPrecision, hardware, comparable), + availablePrecisions: [...new Set(hardwareRows.map((row) => row.precision))].toSorted(), + }; + }); +} + +interface OverviewPrecisionBlock { + comparisonGroups: OverviewComparisonGroup[]; + hardwareStatuses: OverviewHardwareStatus[]; +} + +/** + * Rank one precision's speculative configs: cohorts, per-tier rankings and the + * page-wide hardware statuses. The primary and secondary precisions are both + * built through here, so neither can rank against the other. + */ +function buildPrecisionBlock( + model: Model, + precision: string, + configs: readonly OverviewConfigResult[], + hardwareOrder: readonly OverviewHardwareOrderEntry[], + workloadRows: readonly BenchmarkRow[], +): OverviewPrecisionBlock { + const exclusion = buildExclusion(getModelExclusion(model)); + const cohorts = buildCohorts(exclusion, configs); + const comparable = comparableHardware(exclusion, configs); + const coverage = buildCoverage(hardwareOrder, workloadRows, precision, comparable); + const comparisonGroups = buildComparisonGroups(cohorts, coverage); + // Leadership is a claim against a comparable cohort. Across several cohorts + // there is no model-global leader, so the model row asserts none. + return { + comparisonGroups, + hardwareStatuses: buildHardwareStatuses(coverage, configs, comparisonGroups.length <= 1), + }; +} + +/** + * The other of FP4/FP8. `ranked` (full rows) iff it both ranks a comparable + * cohort AND covers hardware the primary precision lacks; otherwise `coverage` + * (one compact line, ranked rows suppressed). `measuredHardware` always lists + * every hardware measured at this precision so it never disappears from the page. + */ +function buildSecondaryPrecision( + model: Model, + precision: string, + configs: readonly OverviewConfigResult[], + primaryExact: ReadonlySet, + hardwareOrder: readonly OverviewHardwareOrderEntry[], + workloadRows: readonly BenchmarkRow[], +): OverviewSecondaryPrecision | null { + const measuredSet = new Set( + workloadRows.filter((row) => row.precision === precision).map((row) => row.hardware), + ); + if (measuredSet.size === 0) return null; + const measuredHardware = hardwareOrder + .filter(({ hardware }) => measuredSet.has(hardware)) + .map(({ hardwareLabel }) => hardwareLabel); + + const block = buildPrecisionBlock(model, precision, configs, hardwareOrder, workloadRows); + const addsCoverage = [...exactPrimaryHardware(configs)].some((hw) => !primaryExact.has(hw)); + const hasComparable = block.comparisonGroups.some( + (group) => group.primaryRanking.state === 'comparable', + ); + const ranked = hasComparable && addsCoverage; + return { + precision, + state: ranked ? 'ranked' : 'coverage', + comparisonGroups: ranked ? block.comparisonGroups : [], + hardwareStatuses: ranked ? block.hardwareStatuses : [], + measuredHardware, + }; +} + +/** Map one non-ranked hardware's coverage and read to its not-ranked reason. */ +function notRankedReasonFor(status: OverviewHardwareStatus): OverviewNotRankedReason { + switch (status.coverage.kind) { + case 'standard_only': { + return 'standard_decode_only'; + } + case 'alternate_precision_only': { + return 'other_precision_only'; + } + case 'unsupported_precision_only': { + return 'int4_bf16_only'; + } + case 'other_engine_group_only': { + return 'different_serving_cohort'; + } + case 'no_workload_data': { + return 'no_8k1k_data'; + } + case 'comparable_spec': { + // Comparable-spec hardware reaches here only without an exact @50 read. + // Classify by INTERACTIVITY direction, not the boundary name: 'unreachable' + // = tier 50 above the frontier's max, i.e. even the fastest measured point + // is < 50 tok/s/user (the config is too slow, ever) → cannot_reach_at50. + // 'clamped_low' = tier below the frontier's min, i.e. every measured point + // is > 50 tok/s/user — an under-swept sweep gap more concurrency would + // cross, NOT incapability → no_exact_at50. + return status.primary.boundary === 'unreachable' ? 'cannot_reach_at50' : 'no_exact_at50'; + } + } +} + +/** + * One not-ranked entry per page hardware with no visible value — no exact @50 + * read in the primary block and no ranked read in a `ranked` secondary. Reads + * the same coverage/status machinery the blocks use, so it is a derived view + * rather than a parallel source of truth. + */ +function buildNotRanked( + primaryStatuses: readonly OverviewHardwareStatus[], + secondary: OverviewSecondaryPrecision | null, +): OverviewNotRankedEntry[] { + const secondaryRanked = + secondary?.state === 'ranked' + ? new Set( + secondary.hardwareStatuses + .filter((status) => isExactTierRead(status.primary)) + .map((status) => status.hardware), + ) + : new Set(); + + const entries: OverviewNotRankedEntry[] = []; + for (const status of primaryStatuses) { + if (isExactTierRead(status.primary) || secondaryRanked.has(status.hardware)) continue; + entries.push({ + hardware: status.hardware, + hardwareLabel: status.hardwareLabel, + reason: notRankedReasonFor(status), + precisions: status.coverage.availablePrecisions.filter((p) => + OVERVIEW_PRECISIONS.includes(p), + ), + }); + } + return entries; +} + +function emptyModelSummary( + model: Model, + emptyReason: NonNullable, + order: readonly OverviewHardwareOrderEntry[], + workloadRows: readonly BenchmarkRow[], +): OverviewModelSummary { + const hardwareStatuses = buildHardwareStatuses( + buildCoverage(order, workloadRows, null, new Set()), + [], + false, + ); + return { + model, + modelLabel: getModelLabel(model), + selectedPrecision: null, + hardwareStatuses, + comparisonGroups: [], + secondary: null, + notRanked: buildNotRanked(hardwareStatuses, null), + latestWorkloadDate: latestWorkloadDateOf(workloadRows), + emptyReason, + }; +} + +export function buildOverviewModelSummary( + model: Model, + rows: BenchmarkRow[], + hardwareOrder: readonly OverviewHardwareOrderEntry[] = buildOverviewHardwareOrder(rows), +): OverviewModelSummary { + const workloadRows = overviewWorkloadRows(rows); + if (workloadRows.length === 0) { + return emptyModelSummary(model, 'no_8k1k_single_turn_data', hardwareOrder, workloadRows); + } + + if (!workloadRows.some((row) => OVERVIEW_PRECISIONS.includes(row.precision))) { + return emptyModelSummary(model, 'no_fp4_fp8_data', hardwareOrder, workloadRows); + } + + // FP4 and FP8 speculative configs, built once and reused for selection, the + // primary block and the secondary block — the two precisions never blend. + const configsByPrecision = new Map( + OVERVIEW_PRECISIONS.map( + (precision) => [precision, buildPrecisionConfigs(model, workloadRows, precision)] as const, + ), + ); + const selectedPrecision = selectPrimaryFrom(configsByPrecision); + const secondaryPrecision = OVERVIEW_PRECISIONS.find((p) => p !== selectedPrecision)!; + const primaryConfigs = configsByPrecision.get(selectedPrecision)!; + + const primary = buildPrecisionBlock( + model, + selectedPrecision, + primaryConfigs, + hardwareOrder, + workloadRows, + ); + const secondary = buildSecondaryPrecision( + model, + secondaryPrecision, + configsByPrecision.get(secondaryPrecision)!, + exactPrimaryHardware(primaryConfigs), + hardwareOrder, + workloadRows, + ); + + return { + model, + modelLabel: getModelLabel(model), + selectedPrecision, + hardwareStatuses: primary.hardwareStatuses, + comparisonGroups: primary.comparisonGroups, + secondary, + notRanked: buildNotRanked(primary.hardwareStatuses, secondary), + latestWorkloadDate: latestWorkloadDateOf(workloadRows), + emptyReason: null, + }; +} + +/** + * Assemble the whole page from rows already grouped by DISPLAY model. Iterating + * DEFAULT_MODELS fixes the output order and renders every active model — a model + * with no rows falls through to its coverage-only states, exactly as the live + * path does. The page-wide hardware order and dataset date come from the union + * of every model's rows. The live server path and the e2e fixture path feed this + * same function; only the row source differs. + */ +export function assembleOverviewPageData( + rowsByModel: Record, +): OverviewPageData { + const perModel = [...DEFAULT_MODELS].map((model) => ({ model, rows: rowsByModel[model] ?? [] })); + const allRows = perModel.flatMap(({ rows }) => rows); + const hardwareOrder = buildOverviewHardwareOrder(allRows); + return { + models: perModel.map(({ model, rows }) => + buildOverviewModelSummary(model, rows, hardwareOrder), + ), + hardwareOrder, + datasetThroughDate: overviewDatasetThroughDate(allRows), + }; +} diff --git a/packages/app/src/lib/overview-links.test.ts b/packages/app/src/lib/overview-links.test.ts new file mode 100644 index 00000000..0f6c4f35 --- /dev/null +++ b/packages/app/src/lib/overview-links.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest'; + +import { Model, Precision } from './data-mappings'; +import type { OverviewConfigResult, OverviewModelSummary } from './overview-data'; +import { buildOverviewDashboardHref } from './overview-links'; + +const RUN_URL = 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26714221123'; + +/** Query the default fixture produces: one source run, so the run is pinned. */ +const PINNED_QUERY = + 'g_model=Qwen-3.5-397B-A17B&g_rundate=2026-07-18&g_runid=26714221123&i_seq=8k%2F1k' + + '&i_prec=fp4&i_gpus=b200_sglang_mtp&i_spec=mtp&i_disagg=agg&i_optimal=1&i_advlabel=1'; + +function config(overrides: Partial = {}): OverviewConfigResult { + return { + key: 'qwen3.5|b200|sglang|mtp|agg|fp4', + dbModel: 'qwen3.5', + hardware: 'b200', + hardwareLabel: 'B200', + hwKey: 'b200_sglang_mtp', + framework: 'sglang', + frameworkLabel: 'SGLang', + specMethod: 'mtp', + specLabel: 'MTP', + disagg: false, + precision: Precision.FP4, + offloadMode: 'off', + isMultinode: false, + numPrefillGpu: 8, + numDecodeGpu: 8, + totalGpu: 8, + parallelism: 'TP8', + image: 'inferencex/sglang:v0.5.0', + sourceRunUrls: [RUN_URL], + tierValues: [{ tier: 50, value: 1000, boundary: 'interpolated', evidenceDate: null }], + latestDate: '2026-07-18', + oldestFrontierDate: '2026-07-11', + ...overrides, + }; +} + +function summary(overrides: Partial = {}): OverviewModelSummary { + return { + model: Model.Qwen3_5, + modelLabel: 'Qwen 3.5', + selectedPrecision: Precision.FP4, + hardwareStatuses: [], + comparisonGroups: [], + secondary: null, + notRanked: [], + latestWorkloadDate: null, + emptyReason: null, + ...overrides, + }; +} + +describe('buildOverviewDashboardHref', () => { + it('pins model, run, workload and exact configuration on the English route', () => { + expect(buildOverviewDashboardHref('en', summary(), config())).toBe( + `/inference?${PINNED_QUERY}`, + ); + }); + + it('selects the disaggregated deployment mode for a disaggregated configuration', () => { + const href = buildOverviewDashboardHref( + 'en', + summary(), + config({ disagg: true, hwKey: 'gb200_dynamo-trt-disagg_mtp' }), + ); + + expect(href).toBe( + '/inference?g_model=Qwen-3.5-397B-A17B&g_rundate=2026-07-18&g_runid=26714221123' + + '&i_seq=8k%2F1k&i_prec=fp4&i_gpus=gb200_dynamo-trt-disagg_mtp&i_spec=mtp' + + '&i_disagg=disagg&i_optimal=1&i_advlabel=1', + ); + }); + + it('writes g_model even when it equals the dashboard default model', () => { + const href = buildOverviewDashboardHref( + 'en', + summary({ model: Model.DeepSeek_V4_Pro }), + config({ precision: Precision.FP8 }), + ); + + expect(href).toContain('g_model=DeepSeek-V4-Pro'); + expect(href).toContain('i_prec=fp8'); + }); + + it('maps specMethod to the dashboard mtp/stp filter bucket, not the raw DB value', () => { + expect(buildOverviewDashboardHref('en', summary(), config({ specMethod: 'eagle' }))).toContain( + 'i_spec=mtp', + ); + expect(buildOverviewDashboardHref('en', summary(), config({ specMethod: 'none' }))).toContain( + 'i_spec=stp', + ); + expect(buildOverviewDashboardHref('en', summary(), config({ specMethod: 'mtp' }))).toContain( + 'i_spec=mtp', + ); + }); +}); diff --git a/packages/app/src/lib/overview-links.ts b/packages/app/src/lib/overview-links.ts new file mode 100644 index 00000000..8e9deb5e --- /dev/null +++ b/packages/app/src/lib/overview-links.ts @@ -0,0 +1,89 @@ +/** + * @file overview-links.ts + * @description Evidence links for the overview scorecard: a pre-filtered + * /inference view and the CI run a configuration came from. Pure and + * React-free so the server-rendered overview page can build them. + */ + +import { runIdFromRunUrl } from './known-issues'; +import type { OverviewConfigResult, OverviewModelSummary } from './overview-data'; +import type { UrlStateParams } from './url-state'; + +/** The single-turn 8K-in/1K-out workload every overview link filters to. */ +const OVERVIEW_WORKLOAD_SEQ = '8k/1k'; + +/** The `/inference` route base for a locale — shared by every overview link. */ +function inferenceRoute(locale: 'en' | 'zh'): string { + return locale === 'zh' ? '/zh/inference' : '/inference'; +} + +/** + * Maps a raw DB `spec_method` to the dashboard's `SpecMode` filter bucket + * (mirrors `pointSpecMode` in quickFilters.ts, minus its hwKey suffix check — + * overview `specMethod` comes straight from `spec_method`). + */ +function dashboardSpecMode(specMethod: string): 'mtp' | 'stp' { + return specMethod === 'none' || specMethod === '' ? 'stp' : 'mtp'; +} + +/** + * The one run backing a configuration, or null when it has none, has several, + * or its single source URL names no run (a run list rather than a run). Both + * helpers below read this one predicate, so the `g_runid` pin and the source-run + * link can never disagree about whether a single run backs the configuration. + */ +function soleSourceRun(config: OverviewConfigResult): { url: string; id: string } | null { + if (config.sourceRunUrls.length !== 1) return null; + const url = config.sourceRunUrls[0]; + const id = runIdFromRunUrl(url); + return id === null ? null : { url, id }; +} + +/** + * Inference-dashboard link narrowed to the configuration the overview ranked: + * its model, run date, workload, precision, hardware/framework/spec key and + * deployment mode. The run is pinned only when a single run produced the + * configuration — pinning one of several would hide the rest of its frontier. + * + * This is a filtered view, not a proof of topology: `i_gpus` selects a + * hardware/framework/spec key, which can still hold more than one parallelism + * or GPU-count topology. + */ +export function buildOverviewDashboardHref( + locale: 'en' | 'zh', + model: OverviewModelSummary, + config: OverviewConfigResult, +): string { + const params: UrlStateParams = { + g_model: model.model, + g_rundate: config.latestDate, + g_runid: soleSourceRun(config)?.id, + i_seq: OVERVIEW_WORKLOAD_SEQ, + i_prec: config.precision, + i_gpus: config.hwKey, + i_spec: dashboardSpecMode(config.specMethod), + i_disagg: config.disagg ? 'disagg' : 'agg', + i_optimal: '1', + i_advlabel: '1', + }; + + const query = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) query.set(key, value); + } + return `${inferenceRoute(locale)}?${query}`; +} + +/** + * Model-level dashboard view: the workload and precision this page ranked, with + * no configuration pinned. Cohort-level evidence links narrow it further. + */ +export function detailHref(locale: 'en' | 'zh', model: OverviewModelSummary): string { + const query = new URLSearchParams({ + g_model: model.model, + i_seq: OVERVIEW_WORKLOAD_SEQ, + ...(model.selectedPrecision ? { i_prec: model.selectedPrecision } : {}), + i_optimal: '1', + }); + return `${inferenceRoute(locale)}?${query}`; +} diff --git a/packages/app/src/lib/tco-feed.ts b/packages/app/src/lib/tco-feed.ts index 12630d4e..9041b130 100644 --- a/packages/app/src/lib/tco-feed.ts +++ b/packages/app/src/lib/tco-feed.ts @@ -61,6 +61,14 @@ export interface TcoFeedRow { latest_date: string; /** Oldest benchmark date among the frontier knots (staleness flag). */ oldest_frontier_date: string; + /** + * Run dates of the frontier knot(s) actually backing THIS tier read: the + * bracketing pair when interpolated, the single min-interactivity knot when + * clamped_low, null when unreachable. Unlike latest/oldest_frontier_date + * (whole-frontier freshness), it never blends knots outside the tier, so a + * downstream result can carry its own evidence date. Ordered ascending. + */ + evidence_date: { from: string; to: string } | null; } export const DEFAULT_TIERS: readonly number[] = [30, 50, 75, 100]; @@ -197,6 +205,27 @@ interface FrontierPoint { const round3 = (v: number): number => Math.round(v * 1000) / 1000; +/** Two knots' dates as an ascending {from,to} evidence range. */ +function evidenceRange(a: FrontierPoint, b: FrontierPoint): { from: string; to: string } { + return a.date <= b.date ? { from: a.date, to: b.date } : { from: b.date, to: a.date }; +} + +/** + * The frontier knots backing an interpolated read at `tier`: the bracketing + * pair xs[lo] ≤ tier ≤ xs[lo+1], collapsing to a single knot when `tier` lands + * on an endpoint. `frontier` is sorted ascending by interactivity. + */ +function bracketKnots(frontier: FrontierPoint[], tier: number): [FrontierPoint, FrontierPoint] { + const last = frontier.length - 1; + if (tier <= frontier[0].interactivity) return [frontier[0], frontier[0]]; + if (tier >= frontier[last].interactivity) return [frontier[last], frontier[last]]; + let lo = 0; + for (let i = 0; i < last; i++) { + if (frontier[i].interactivity <= tier) lo = i; + } + return [frontier[lo], frontier[lo + 1]]; +} + /** * Compute the feed: for each (workload, hardware), build the * (interactivity, output tok/s/GPU) Pareto frontier across every config — @@ -268,16 +297,22 @@ export function computeTcoFeed( for (const tier of tiers) { let value: number; let boundary: TcoTierBoundary; + let evidenceDate: { from: string; to: string } | null; if (tier > maxIv) { value = 0; boundary = 'unreachable'; + evidenceDate = null; } else if (tier < minIv) { value = ys[0]; boundary = 'clamped_low'; + // The clamped read is the min-interactivity knot's throughput. + evidenceDate = evidenceRange(frontier[0], frontier[0]); } else { const raw = hermiteInterpolate(xs, ys, slopes, tier); value = Math.max(yLo, Math.min(yHi, raw)); boundary = 'interpolated'; + const [lo, hi] = bracketKnots(frontier, tier); + evidenceDate = evidenceRange(lo, hi); } out.push({ hardware, @@ -290,6 +325,7 @@ export function computeTcoFeed( frontier_max_interactivity: round3(maxIv), latest_date: latest, oldest_frontier_date: oldest, + evidence_date: evidenceDate, }); } } diff --git a/packages/app/src/lib/toggle-set.ts b/packages/app/src/lib/toggle-set.ts new file mode 100644 index 00000000..963d3ea4 --- /dev/null +++ b/packages/app/src/lib/toggle-set.ts @@ -0,0 +1,32 @@ +/** + * Pure toggle state transition. + * + * - If all items are active and one is toggled, only that item remains active (solo). + * - If only one item is active and it's toggled, all items become active (restore all). + * - Otherwise, the item is simply added or removed from the active set. + * + * Kept free of React so server-only modules (`exclusion.ts`, and through it the + * overview data layer) can reuse it without pulling client hooks into a Server + * Component graph. + */ +export function computeToggle(prev: Set, item: string, allItems: Set): Set { + const allAreActive = prev.size === allItems.size; + const isActive = prev.has(item); + + if (isActive) { + if (allAreActive) { + // Solo: only keep the clicked item + return new Set([item]); + } else if (prev.size === 1) { + // Restore all: re-enable everything + return allItems; + } + // Remove the clicked item + const next = new Set(prev); + next.delete(item); + return next; + } + // Add the clicked item + const next = new Set([...prev, item]); + return next; +} From 1febe16bdc04899c6b3f7206f92130ee401be716 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 14:23:05 -0700 Subject: [PATCH 02/13] feat(overview): infinity placeholder chips for missing results --- packages/app/cypress/e2e/overview.cy.ts | 16 +++++++--- .../overview/overview-scorecard.tsx | 32 ++++++++++++++----- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index bdd92f69..1b258f26 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -89,11 +89,12 @@ describe('Overview page', () => { cy.contains('Jul 18').should('exist'); cy.contains('381').should('exist'); cy.contains('Only exact result').should('exist'); - // Every remaining hardware carries one reason, both clamp directions included. - cy.contains('cannot reach @50').should('exist'); - cy.contains('no exact @50 result').should('exist'); - cy.contains('no 8K/1K data').should('exist'); - cy.contains('standard decode only').should('exist'); + // Every remaining hardware carries an ∞ chip whose title/aria holds the + // reason, both clamp directions included; ∞ never renders with a percent. + cy.get('[title="cannot reach @50"]').should('exist').and('contain', '∞'); + cy.get('[title="no exact @50 result"]').should('exist').and('contain', '∞'); + cy.get('[title="no 8K/1K data"]').should('exist').and('contain', '∞'); + cy.get('[title="standard decode only"]').should('exist').and('contain', '∞'); // Each ranked value links into its own pre-filtered dashboard view. cy.get('a[href*="i_gpus="]') .first() @@ -102,6 +103,11 @@ describe('Overview page', () => { .and('include', 'i_gpus=b300_sglang_mtp') .and('include', 'i_spec=mtp'); }); + // The legend lives once in the page footnotes; ∞ never renders as a percent. + cy.contains('∞ = no comparable result').should('exist'); + cy.get('body') + .invoke('text') + .should('not.match', /∞\s*%/); }); it('opens a ranked secondary only when the other precision adds comparable hardware', () => { diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index a1c02907..f80c1d07 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -65,7 +65,7 @@ export const OVERVIEW_STRINGS = { `Open filtered dashboard: ${model} · ${config}`, evidenceHighAria: (model: string, config: string) => `Open filtered dashboard @${OVERVIEW_HIGH_TIER} tok/s/user leader: ${model} · ${config}`, - notRankedLine: (entries: string) => `Not ranked: ${entries}`, + infinityLegend: '∞ = no comparable result', notRankedReasons: { standard_decode_only: 'standard decode only', int4_bf16_only: 'INT4/BF16 only', @@ -125,7 +125,7 @@ export const OVERVIEW_STRINGS = { `打开筛选后的仪表板:${model} · ${config}`, evidenceHighAria: (model: string, config: string) => `打开筛选后的仪表板 @${OVERVIEW_HIGH_TIER} tok/s/user:${model} · ${config}`, - notRankedLine: (entries: string) => `未参与排名:${entries}`, + infinityLegend: '∞ = 无可比结果', notRankedReasons: { standard_decode_only: '仅标准解码', int4_bf16_only: '仅 INT4/BF16', @@ -551,12 +551,27 @@ function NotRankedLine({ // coverage 'measured' list — is not repeated here; dedup against the secondary's // measured labels so each hardware surfaces in exactly one place. const secondaryMeasured = new Set(model.secondary?.measuredHardware); - const entries = model.notRanked - .filter((entry) => !secondaryMeasured.has(entry.hardwareLabel)) - .map((entry) => `${entry.hardwareLabel} — ${notRankedReason(entry, strings)}`) - .join(' · '); - if (entries === '') return null; - return

{strings.notRankedLine(entries)}

; + const entries = model.notRanked.filter((entry) => !secondaryMeasured.has(entry.hardwareLabel)); + if (entries.length === 0) return null; + // ∞ marks missing/unavailable only — it never enters ranking or gap math, so it + // must never render with a percent. The reason rides in title/aria; the page + // footer carries the legend. + return ( +

+ {entries.map((entry) => { + const reason = notRankedReason(entry, strings); + return ( + + {entry.hardwareLabel} + + ); + })} +

+ ); } /** The full per-model hierarchy, identical on both surfaces. */ @@ -729,6 +744,7 @@ export function OverviewMethodology({ strings }: { strings: OverviewStrings }) { return (

{strings.methodologyNote}

+

{strings.infinityLegend}

{strings.cohortNote}

{strings.interpolationNote}

From 76af0922a528684161ab9d0e009ddfe976deeff9 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 18:33:18 -0700 Subject: [PATCH 03/13] feat(overview): per-platform matrix with tier switcher and stack labels --- packages/app/cypress/e2e/overview.cy.ts | 335 +++++-- .../cypress/fixtures/api/overview-rows.json | 39 +- packages/app/src/app/overview/page.tsx | 12 +- packages/app/src/app/zh/overview/page.tsx | 12 +- .../src/components/overview/overview-page.tsx | 4 +- .../overview/overview-scorecard.tsx | 842 ++++++---------- packages/app/src/lib/overview-data.server.ts | 18 +- packages/app/src/lib/overview-data.test.ts | 871 +++++++---------- packages/app/src/lib/overview-data.ts | 905 +++++------------- packages/app/src/lib/overview-links.test.ts | 18 +- packages/app/src/lib/overview-links.ts | 22 +- packages/app/src/lib/tco-feed.test.ts | 28 + packages/app/src/lib/tco-feed.ts | 1 + 13 files changed, 1239 insertions(+), 1868 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 1b258f26..89fb5039 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -1,11 +1,3 @@ -// Focused smoke coverage for /overview. In fixtures mode the page is served by -// the synthetic cypress/fixtures/api/overview-rows.json through the real data -// builder, so every expected value below was derived by running the assembler -// over that fixture (the drift guard in overview-data.test.ts locks the same -// values) — never eyeballed. The fixture exercises every 8.7 state: coverage-driven -// primary precision, ranked vs coverage secondary, all not-ranked reasons, and — -// uniquely — a cross-day evidence range the live dataset does not produce. - const MODEL_LABELS = [ 'DeepSeek V4 Pro 1.6T', 'Kimi K2.5/2.6/2.7-Code 1T', @@ -14,20 +6,22 @@ const MODEL_LABELS = [ 'Qwen3.5 397B', ]; -/** The page must never scroll sideways: the whole comparison has to fit. */ +const PLATFORM_HEADERS = [ + 'Model', + 'B200', + 'MI355X', + 'B300', + 'GB200 NVL72', + 'GB300 NVL72', + 'Details', +]; + function expectNoHorizontalOverflow() { cy.document().then((doc) => { expect(doc.documentElement.scrollWidth).to.be.lte(doc.documentElement.clientWidth); }); } -/** - * A narrow document is not enough: a table parked in an `overflow-x` region - * keeps the document narrow while still scrolling sideways, so nothing inside - * the surface may be wider than its own box either. `sr-only` clips (1px) and - * non-replaced inline boxes are exempt — CSSOM reports a spurious `scrollWidth` - * for inline boxes in Firefox, and `overflow` cannot apply to them anyway. - */ function expectNoHorizontalScroller(testId: string) { cy.get(`[data-testid="${testId}"]`).then(([surface]) => { const scrollers = [surface, ...surface.querySelectorAll('*')] @@ -46,112 +40,275 @@ function desktopModel(model: string) { return cy.get(`[data-testid="overview-desktop-model"][data-model="${model}"]`); } +function mobileModel(model: string) { + return cy.get(`[data-testid="overview-mobile-model"][data-model="${model}"]`); +} + +function pair(pairId: string) { + return cy.get(`[data-testid="overview-pair"][data-pair="${pairId}"]`); +} + describe('Overview page', () => { - it('summarizes every active model without interactive widgets', () => { - cy.viewport(1280, 800); + it('renders the full platform matrix for every active model', () => { + cy.viewport(1280, 900); cy.visit('/overview'); cy.contains('h1', 'AI Inference Overview').should('exist'); - cy.get('[data-testid="overview-desktop-matrix"]').should('be.visible'); - cy.get('[data-testid="overview-desktop-matrix"] h2').should('have.length', MODEL_LABELS.length); + cy.contains( + 'Every active model across MI355X, B200, B300, GB200 and GB300 at a glance.', + ).should('exist'); + cy.contains('Best validated stack per platform').should('exist'); + cy.contains('Database snapshot through Jul 18').should('exist'); + cy.get('[data-testid="overview-desktop-matrix"]') + .should('be.visible') + .within(() => { + cy.get('thead th').then(($headers) => { + expect([...$headers].map((header) => header.textContent?.trim())).to.deep.equal( + PLATFORM_HEADERS, + ); + }); + cy.get('[data-testid="overview-desktop-model"]').should('have.length', MODEL_LABELS.length); + cy.get('[data-testid="overview-baseline"]').should('have.length', MODEL_LABELS.length); + cy.get('[data-testid="overview-pair"]').should('have.length', MODEL_LABELS.length * 4); + cy.get('details, summary, button').should('not.exist'); + cy.contains(/PRIMARY|Ranked results/).should('not.exist'); + }); for (const label of MODEL_LABELS) { cy.get('[data-testid="overview-desktop-matrix"]').should('contain.text', label); } - // Database-wide freshness line + the methodology footnote's once-per-page note. - cy.contains('Database snapshot through Jul 18').should('exist'); - cy.contains( - 'The default precision maximizes comparable hardware coverage. Not ranked does not mean slower.', - ).should('exist'); - // A one-glance summary, not an interactive widget. - cy.get('[data-testid="overview-desktop-matrix"]').within(() => { - cy.get('details, summary, button').should('not.exist'); + }); + + it('shows per-cell best reads with precision badges, dates, links, and matched deltas', () => { + cy.viewport(1280, 900); + cy.visit('/overview'); + + desktopModel('Qwen-3.5-397B-A17B').within(() => { + cy.get('[data-testid="overview-baseline"][data-hardware="b200"]') + .should('contain.text', '900') + .and('contain.text', 'FP8'); + pair('mi355x-vs-b200').within(() => { + cy.contains('SGLang · FP8').should('exist'); + cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"]') + .should('contain.text', '760') + .find('a') + .should('have.attr', 'title', 'MI355X · SGLang · FP8 · MTP'); + cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"] a') + .should('have.attr', 'href') + .and('include', 'g_model=Qwen-3.5-397B-A17B') + .and('include', 'i_prec=fp8') + .and('include', 'i_gpus=mi355x_sglang_mtp'); + cy.get('[data-testid="overview-pair-evidence-date"][data-hardware="mi355x"]').should( + 'have.text', + 'Jul 18', + ); + cy.get('[data-testid="overview-pair-delta"]').should('have.text', '−16% vs B200'); + cy.contains('At 100, MI355X leads').should('exist'); + }); + }); + + desktopModel('DeepSeek-V4-Pro').within(() => { + cy.get('[data-testid="overview-baseline"][data-hardware="b200"]') + .should('contain.text', '900') + .and('contain.text', 'FP4'); + pair('b300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="b300"]').should( + 'contain.text', + '1,122', + ); + cy.get('[data-testid="overview-pair-evidence-date"][data-hardware="b300"]').should( + 'have.text', + 'Jun 24–Jul 4', + ); + cy.get('[data-testid="overview-pair-delta"]').should('have.text', '+25% vs B200'); + }); + }); + }); + + it('never mixes precisions or releases in a delta and says so instead', () => { + cy.viewport(1280, 900); + cy.visit('/overview'); + + desktopModel('DeepSeek-V4-Pro').within(() => { + pair('gb200-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="gb200"]') + .should('contain.text', '600') + .parents('[data-testid="overview-pair"]') + .should('contain.text', 'FP8'); + cy.get('[data-testid="overview-pair-mismatch"]').should( + 'have.text', + 'FP8 vs FP4 · no comparable delta', + ); + cy.get('[data-testid="overview-pair-delta"]').should('not.exist'); + }); + }); + + desktopModel('Qwen-3.5-397B-A17B').within(() => { + pair('b300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="b300"]').should( + 'contain.text', + '1,151', + ); + cy.get('[data-testid="overview-pair-mismatch"]').should( + 'have.text', + 'FP4 vs FP8 · no comparable delta', + ); + cy.get('[data-testid="overview-pair-delta"]').should('not.exist'); + }); }); }); - it('ranks the primary precision, dates each read, and accounts for every hardware', () => { - cy.viewport(1280, 800); + it('distinguishes candidate, baseline, and whole-row missing results without a percent', () => { + cy.viewport(1280, 900); cy.visit('/overview'); desktopModel('DeepSeek-V4-Pro').within(() => { - // Primary precision, its leader, and the runner-up's own signed delta (ASCII minus). - cy.contains('FP4 · PRIMARY @50').should('exist'); - cy.contains('B300').should('exist'); - cy.contains('1,122').should('exist'); - cy.contains('Leader').should('exist'); - cy.contains('-20%').should('exist'); - // FP8 measured one platform → one compact coverage line, not a second table. - cy.contains('FP8 coverage: GB200 NVL72 measured; insufficient comparable results.').should( - 'exist', - ); - // Leader @50 is bracketed by two run days (en-dash range); same-day reads and - // the @100 read (its own leader) collapse to a single date. - cy.contains('Jun 24–Jul 4').should('exist'); - cy.contains('Jul 18').should('exist'); - cy.contains('381').should('exist'); - cy.contains('Only exact result').should('exist'); - // Every remaining hardware carries an ∞ chip whose title/aria holds the - // reason, both clamp directions included; ∞ never renders with a percent. - cy.get('[title="cannot reach @50"]').should('exist').and('contain', '∞'); - cy.get('[title="no exact @50 result"]').should('exist').and('contain', '∞'); - cy.get('[title="no 8K/1K data"]').should('exist').and('contain', '∞'); - cy.get('[title="standard decode only"]').should('exist').and('contain', '∞'); - // Each ranked value links into its own pre-filtered dashboard view. - cy.get('a[href*="i_gpus="]') - .first() - .should('have.attr', 'href') - .and('include', 'g_model=DeepSeek-V4-Pro') - .and('include', 'i_gpus=b300_sglang_mtp') - .and('include', 'i_spec=mtp'); + pair('mi355x-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-missing"][data-hardware="mi355x"]') + .should('contain.text', '∞') + .and('have.attr', 'title', 'no exact @50 result'); + }); + pair('gb300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-missing"][data-hardware="gb300"]') + .should('contain.text', '∞') + .and('have.attr', 'title', 'cannot reach @50'); + }); + }); + + desktopModel('MiniMax-M3').within(() => { + cy.get('[data-testid="overview-baseline"] [data-testid="overview-pair-missing"]') + .should('contain.text', '∞') + .and('have.attr', 'title', 'no 8K/1K data'); + pair('gb300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="gb300"]').should( + 'contain.text', + '700', + ); + cy.get('[data-testid="overview-pair-delta"]').should('not.exist'); + cy.get('[data-testid="overview-pair-mismatch"]').should('not.exist'); + }); + }); + + desktopModel('Kimi-K2.5').within(() => { + cy.get('[data-testid="overview-pair-missing"]').should('have.length', 5); + cy.get('[data-testid="overview-pair-value"]').should('not.exist'); }); - // The legend lives once in the page footnotes; ∞ never renders as a percent. cy.contains('∞ = no comparable result').should('exist'); cy.get('body') .invoke('text') .should('not.match', /∞\s*%/); }); - it('opens a ranked secondary only when the other precision adds comparable hardware', () => { - cy.viewport(1280, 800); + it('re-renders the whole matrix at the service level the URL names, via plain links', () => { + cy.viewport(1280, 900); cy.visit('/overview'); - // Qwen: FP8 ranks a comparable pair AND adds MI355X, so it renders subordinate rows. + cy.get('[data-testid="overview-tier-switcher"]').within(() => { + cy.get('[aria-current="page"]').should('have.text', '50'); + cy.get('a').should('have.length', 3); + cy.contains('a', '30').should('have.attr', 'href', '/overview?tier=30'); + cy.contains('a', '100').should('have.attr', 'href', '/overview?tier=100').click(); + }); + + cy.location('search').should('eq', '?tier=100'); + cy.contains('Output tok/s/GPU @100 tok/s/user').should('exist'); + cy.get('[data-testid="overview-tier-switcher"]').within(() => { + cy.get('[aria-current="page"]').should('have.text', '100'); + cy.contains('a', '50').should('have.attr', 'href', '/overview'); + }); + desktopModel('Qwen-3.5-397B-A17B').within(() => { - cy.contains('FP4 · PRIMARY @50').should('exist'); - cy.contains('FP8 @50').should('exist'); - cy.contains('MI355X').should('exist'); - cy.contains('760').should('exist'); - cy.contains('-16%').should('exist'); + cy.get('[data-testid="overview-baseline"][data-hardware="b200"]') + .should('contain.text', '432') + .and('contain.text', 'FP8'); + pair('mi355x-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"]').should( + 'contain.text', + '635', + ); + cy.get('[data-testid="overview-pair-delta"]').should('have.text', '+47% vs B200'); + cy.contains('At 100, MI355X leads').should('not.exist'); + }); + pair('b300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-missing"][data-hardware="b300"]') + .should('contain.text', '∞') + .and('have.attr', 'title', 'cannot reach @100'); + }); }); - // MiniMax: wider exact-@50 FP8 coverage flips the primary precision to FP8; - // FP4 falls back to a single-hardware coverage line. - desktopModel('MiniMax-M3').within(() => { - cy.contains('FP8 · PRIMARY @50').should('exist'); - cy.contains('FP4 coverage: H200 measured; insufficient comparable results.').should('exist'); + + cy.visit('/overview?tier=30'); + desktopModel('DeepSeek-V4-Pro').within(() => { + pair('b300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="b300"]').should( + 'contain.text', + '1,300', + ); + cy.get('[data-testid="overview-pair-delta"]').should('not.exist'); + }); + cy.get('[data-testid="overview-baseline"] [data-testid="overview-pair-missing"]') + .should('contain.text', '∞') + .and('have.attr', 'title', 'no exact @30 result'); }); + cy.get('body') + .invoke('text') + .should('not.match', /∞\s*%/); }); - it('stacks on a 390px phone with no sideways scroll', () => { - cy.viewport(390, 844); - cy.visit('/overview'); + it('uses the same cell semantics on mobile and fits both 390px and 320px widths', () => { + for (const width of [390, 320]) { + cy.viewport(width, 844); + cy.visit('/overview'); - cy.get('[data-testid="overview-mobile-list"]').should('be.visible'); - cy.get('[data-testid="overview-desktop-matrix"]').should('not.be.visible'); - cy.get('[data-testid="overview-mobile-list"]').within(() => { - cy.get('details, summary, button').should('not.exist'); - }); - expectNoHorizontalOverflow(); - expectNoHorizontalScroller('overview-mobile-list'); + cy.get('[data-testid="overview-mobile-list"]').should('be.visible'); + cy.get('[data-testid="overview-tier-switcher"]').should('be.visible'); + cy.get('[data-testid="overview-desktop-matrix"]').should('not.be.visible'); + mobileModel('Qwen-3.5-397B-A17B').within(() => { + cy.get('[data-testid="overview-pair"]').should('have.length', 4); + pair('mi355x-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-value"][data-hardware="mi355x"]').should( + 'contain.text', + '760', + ); + cy.get('[data-testid="overview-pair-delta"]').should('have.text', '−16% vs B200'); + }); + }); + expectNoHorizontalOverflow(); + expectNoHorizontalScroller('overview-mobile-list'); + } }); - it('renders the Chinese sibling with the same hierarchy and the per-GPU unit', () => { - cy.viewport(1280, 800); + it('renders the Chinese sibling with equivalent matrix copy and semantics', () => { + cy.viewport(1280, 900); cy.visit('/zh/overview'); cy.contains('h1', 'AI 推理总览').should('exist'); - cy.contains('输出 tok/s/GPU').should('exist'); - desktopModel('DeepSeek-V4-Pro').within(() => { - cy.contains('FP4 · 主排名 @50').should('exist'); + cy.contains('一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。').should( + 'exist', + ); + cy.contains('各平台最佳验证配置').should('exist'); + desktopModel('Qwen-3.5-397B-A17B').within(() => { + pair('mi355x-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-delta"]').should('have.text', '相对 B200 −16%'); + cy.contains('100 档由 MI355X 领先').should('exist'); + }); + pair('b300-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-mismatch"]').should( + 'have.text', + 'FP4 与 FP8 · 无可比差值', + ); + }); + }); + cy.contains('∞ = 无可比结果').should('exist'); + + cy.visit('/zh/overview?tier=100'); + cy.contains('每 GPU 输出 tok/s @100 tok/s/用户').should('exist'); + desktopModel('Qwen-3.5-397B-A17B').within(() => { + pair('mi355x-vs-b200').within(() => { + cy.get('[data-testid="overview-pair-delta"]').should('have.text', '相对 B200 +47%'); + cy.contains('100 档由 MI355X 领先').should('not.exist'); + }); + }); + cy.get('[data-testid="overview-tier-switcher"]').within(() => { + cy.contains('a', '50').should('have.attr', 'href', '/zh/overview'); }); - cy.contains('默认精度优先覆盖更多可比较硬件。未参与排名不代表性能更低。').should('exist'); }); }); diff --git a/packages/app/cypress/fixtures/api/overview-rows.json b/packages/app/cypress/fixtures/api/overview-rows.json index d0e53d8a..96a441b0 100644 --- a/packages/app/cypress/fixtures/api/overview-rows.json +++ b/packages/app/cypress/fixtures/api/overview-rows.json @@ -1,26 +1,29 @@ { "DeepSeek-V4-Pro": [ - {"id":1,"hardware":"b300","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":1300},"date":"2026-06-24","run_url":null}, - {"id":2,"hardware":"b300","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":70,"output_tput_per_gpu":1000},"date":"2026-07-04","run_url":null}, - {"id":3,"hardware":"b200","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":40,"output_tput_per_gpu":1050},"date":"2026-07-18","run_url":null}, - {"id":4,"hardware":"b200","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":24,"offload_mode":"off","image":null,"metrics":{"median_intvty":110,"output_tput_per_gpu":300},"date":"2026-07-18","run_url":null}, - {"id":5,"hardware":"mi355x","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":60,"output_tput_per_gpu":760},"date":"2026-07-18","run_url":null}, - {"id":6,"hardware":"gb300","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":45,"output_tput_per_gpu":1050},"date":"2026-07-18","run_url":null}, - {"id":7,"hardware":"h100","framework":"sglang","model":"dsv4","precision":"fp4","spec_method":"none","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":800},"date":"2026-07-18","run_url":null}, - {"id":8,"hardware":"gb200","framework":"sglang","model":"dsv4","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":600},"date":"2026-07-18","run_url":null} + {"id": 1, "hardware": "b300", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 1300}, "date": "2026-06-24", "run_url": null}, + {"id": 2, "hardware": "b300", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 70, "output_tput_per_gpu": 1000}, "date": "2026-07-04", "run_url": null}, + {"id": 3, "hardware": "b200", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 40, "output_tput_per_gpu": 1050}, "date": "2026-07-18", "run_url": null}, + {"id": 4, "hardware": "b200", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 110, "output_tput_per_gpu": 300}, "date": "2026-07-18", "run_url": null}, + {"id": 5, "hardware": "mi355x", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 60, "output_tput_per_gpu": 760}, "date": "2026-07-18", "run_url": null}, + {"id": 6, "hardware": "gb300", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 45, "output_tput_per_gpu": 1050}, "date": "2026-07-18", "run_url": null}, + {"id": 7, "hardware": "h100", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "none", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 800}, "date": "2026-07-18", "run_url": null}, + {"id": 8, "hardware": "gb200", "framework": "sglang", "model": "dsv4", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null} ], "Qwen-3.5-397B-A17B": [ - {"id":9,"hardware":"b300","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":1400},"date":"2026-07-18","run_url":null}, - {"id":10,"hardware":"b300","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":70,"output_tput_per_gpu":980},"date":"2026-07-18","run_url":null}, - {"id":11,"hardware":"b200","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":40,"output_tput_per_gpu":1150},"date":"2026-07-18","run_url":null}, - {"id":12,"hardware":"b200","framework":"sglang","model":"qwen3.5","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":24,"offload_mode":"off","image":null,"metrics":{"median_intvty":80,"output_tput_per_gpu":720},"date":"2026-07-18","run_url":null}, - {"id":13,"hardware":"b200","framework":"sglang","model":"qwen3.5","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":900},"date":"2026-07-18","run_url":null}, - {"id":14,"hardware":"mi355x","framework":"sglang","model":"qwen3.5","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":760},"date":"2026-07-18","run_url":null} + {"id": 9, "hardware": "b300", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 1400}, "date": "2026-07-18", "run_url": null}, + {"id": 10, "hardware": "b300", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 70, "output_tput_per_gpu": 980}, "date": "2026-07-18", "run_url": null}, + {"id": 11, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 40, "output_tput_per_gpu": 800}, "date": "2026-07-18", "run_url": null}, + {"id": 12, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 80, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null}, + {"id": 13, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 900}, "date": "2026-07-18", "run_url": null}, + {"id": 14, "hardware": "mi355x", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 760}, "date": "2026-07-18", "run_url": null}, + {"id": 19, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 120, "output_tput_per_gpu": 300}, "date": "2026-07-18", "run_url": null}, + {"id": 20, "hardware": "mi355x", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 120, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null} ], "MiniMax-M3": [ - {"id":15,"hardware":"h200","framework":"sglang","model":"minimaxm3","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":8,"offload_mode":"off","image":null,"metrics":{"median_intvty":30,"output_tput_per_gpu":820},"date":"2026-07-18","run_url":null}, - {"id":16,"hardware":"h200","framework":"sglang","model":"minimaxm3","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":16,"offload_mode":"off","image":null,"metrics":{"median_intvty":70,"output_tput_per_gpu":560},"date":"2026-07-18","run_url":null}, - {"id":17,"hardware":"mi325x","framework":"sglang","model":"minimaxm3","precision":"fp8","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":650},"date":"2026-07-18","run_url":null}, - {"id":18,"hardware":"h200","framework":"sglang","model":"minimaxm3","precision":"fp4","spec_method":"mtp","disagg":false,"is_multinode":false,"prefill_tp":8,"prefill_ep":1,"prefill_dp_attention":false,"prefill_num_workers":1,"decode_tp":8,"decode_ep":1,"decode_dp_attention":false,"decode_num_workers":1,"num_prefill_gpu":8,"num_decode_gpu":8,"benchmark_type":"single_turn","isl":8192,"osl":1024,"conc":12,"offload_mode":"off","image":null,"metrics":{"median_intvty":50,"output_tput_per_gpu":600},"date":"2026-07-18","run_url":null} + {"id": 15, "hardware": "h200", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 820}, "date": "2026-07-18", "run_url": null}, + {"id": 16, "hardware": "h200", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 70, "output_tput_per_gpu": 560}, "date": "2026-07-18", "run_url": null}, + {"id": 17, "hardware": "mi325x", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 650}, "date": "2026-07-18", "run_url": null}, + {"id": 18, "hardware": "h200", "framework": "sglang", "model": "minimaxm3", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null}, + {"id": 21, "hardware": "gb300", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 700}, "date": "2026-07-18", "run_url": null} ] } diff --git a/packages/app/src/app/overview/page.tsx b/packages/app/src/app/overview/page.tsx index e76d0d35..8e1bde73 100644 --- a/packages/app/src/app/overview/page.tsx +++ b/packages/app/src/app/overview/page.tsx @@ -4,12 +4,13 @@ import { SITE_NAME, SITE_URL } from '@semianalysisai/inferencex-constants'; import { OverviewPageContent } from '@/components/overview/overview-page'; import { enAlternates } from '@/lib/i18n'; +import { resolveOverviewTier } from '@/lib/overview-data'; import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; const DESCRIPTION = - 'Comparable validated AI inference serving results for every active model and platform at a fixed single-turn 8K input / 1K output workload, ranked at 50 tok/s/user.'; + 'Every active model across MI355X, B200, B300, GB200 and GB300 at a fixed single-turn 8K input / 1K output workload — each platform’s best validated speculative-decode serving result at 50 tok/s/user, with same-precision deltas against B200.'; export const metadata: Metadata = { title: 'AI Inference Overview', @@ -28,8 +29,13 @@ export const metadata: Metadata = { }, }; -export default async function OverviewPage() { - const data = await getOverviewPageData(); +interface Props { + searchParams: Promise>; +} + +export default async function OverviewPage({ searchParams }: Props) { + const sp = await searchParams; + const data = await getOverviewPageData(resolveOverviewTier(sp.tier)); return (
diff --git a/packages/app/src/app/zh/overview/page.tsx b/packages/app/src/app/zh/overview/page.tsx index 91e4fd48..dd5d6cc9 100644 --- a/packages/app/src/app/zh/overview/page.tsx +++ b/packages/app/src/app/zh/overview/page.tsx @@ -4,12 +4,13 @@ import { SITE_NAME, SITE_URL } from '@semianalysisai/inferencex-constants'; import { OverviewPageContent } from '@/components/overview/overview-page'; import { ZH_OG_LOCALE, zhAlternates } from '@/lib/i18n'; +import { resolveOverviewTier } from '@/lib/overview-data'; import { getOverviewPageData } from '@/lib/overview-data.server'; export const dynamic = 'force-dynamic'; const DESCRIPTION = - '以固定单轮 8K 输入 / 1K 输出负载,按 50 tok/s/user 档位排名,对比每个活跃模型在各硬件平台上可比的已验证服务结果。'; + '在固定单轮 8K 输入 / 1K 输出负载下,总览各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现;每格为该平台最佳验证推测解码结果(50 tok/s/user 档位),相对 B200 的差值仅在同精度结果之间计算。'; export const metadata: Metadata = { title: 'AI 推理总览', @@ -29,8 +30,13 @@ export const metadata: Metadata = { }, }; -export default async function ZhOverviewPage() { - const data = await getOverviewPageData(); +interface Props { + searchParams: Promise>; +} + +export default async function ZhOverviewPage({ searchParams }: Props) { + const sp = await searchParams; + const data = await getOverviewPageData(resolveOverviewTier(sp.tier)); return (
diff --git a/packages/app/src/components/overview/overview-page.tsx b/packages/app/src/components/overview/overview-page.tsx index c99f3ae6..5ac8c864 100644 --- a/packages/app/src/components/overview/overview-page.tsx +++ b/packages/app/src/components/overview/overview-page.tsx @@ -5,6 +5,7 @@ import { DesktopOverviewMatrix, MobileOverviewList, OverviewMethodology, + OverviewTierSwitcher, overviewFormatters, OVERVIEW_STRINGS, type OverviewLocale, @@ -31,11 +32,12 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) { {strings.purpose}

- {strings.scope} + {strings.scope(data.tier)}

{snapshot === null ? null : (

{snapshot}

)} + {/* Official-only summary; uploaded runs remain in the linked dashboard. */} diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index f80c1d07..b5d09806 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -1,149 +1,100 @@ /** * @file overview-scorecard.tsx - * @description The comparison surface itself. Desktop (a matrix at `xl`) and - * mobile (stacked cards below `xl`) render the SAME per-model hierarchy from one - * `ModelHierarchy` component, so the two surfaces cannot drift semantically. + * @description The comparison surface itself: a portfolio matrix of every + * active model (rows) across today's key serving platforms (columns — B200 as + * the reference, then MI355X, B300, GB200 NVL72, GB300 NVL72). The desktop + * table at `xl` and the mobile cards below `xl` render every cell through the + * SAME `BaselineCell`/`CandidateCell` components, so the two surfaces cannot + * drift semantically. * - * Each model reads top to bottom as: the primary precision's ranked hardware as - * horizontal chips (@50), the secondary precision (a subordinate ranked block or - * one compact coverage line), the @100 capability read, then a not-ranked line - * that accounts for every remaining hardware with an explicit reason. Precisions - * and incompatible cohorts are never compared across — each ranks only itself. + * Each cell shows that platform's own best validated speculative read at the + * displayed tier (50 tok/s/user by default; `?tier=` picks another) with its + * precision badge, evidence date and filtered-dashboard link. The signed + * delta against B200 appears only when both displayed reads share a precision + * and a release — mismatches say so instead of comparing FP4 with FP8. A side + * with no exact read shows `∞` with its reason, never a number or percent. */ -import { Badge } from '@/components/ui/badge'; import { - overviewHighValue, OVERVIEW_HIGH_TIER, - OVERVIEW_PRIMARY_TIER, - type OverviewComparisonGroup, - type OverviewConfigResult, - type OverviewHardwareStatus, + OVERVIEW_TIERS, + type OverviewHeadlinePairComparison, + type OverviewHeadlinePairMember, type OverviewModelSummary, - type OverviewNotRankedEntry, + type OverviewTier, } from '@/lib/overview-data'; -import { detailHref } from '@/lib/overview-links'; +import { buildOverviewDashboardHref, detailHref, overviewTierHref } from '@/lib/overview-links'; -import { OverviewDetailLink, OverviewDashboardLink } from './overview-detail-link'; +import { OverviewDetailLink } from './overview-detail-link'; export type OverviewLocale = 'en' | 'zh'; export const OVERVIEW_STRINGS = { en: { title: 'AI Inference Overview', - purpose: 'Compare validated serving results across active models and hardware.', - scope: `8K→1K · Single-turn · Output tok/s/GPU at ${OVERVIEW_PRIMARY_TIER} tok/s/user · Speculative decode only · FP4/FP8 by comparable coverage`, + purpose: 'Every active model across MI355X, B200, B300, GB200 and GB300 at a glance.', + scope: (tier: number) => + `8K→1K · Single-turn · Output tok/s/GPU @${tier} tok/s/user · Speculative decode only · Best validated stack per platform`, + tierNavLabel: 'Service level', + tierUnit: 'tok/s/user', snapshot: (through: string) => `Database snapshot through ${through}`, + caption: + "Best validated serving results for every active model across today's key platforms, each compared with B200.", modelHeader: 'Model', - resultsHeader: 'Ranked results', detailsHeader: 'Details', - caption: `Validated output tok/s/GPU per model and comparable cohort at ${OVERVIEW_PRIMARY_TIER} and ${OVERVIEW_HIGH_TIER} tok/s/user.`, - cohort: (engine: string, mode: string) => `${engine} · ${mode}`, - aggregated: 'Aggregated', - disaggregated: 'Disaggregated', - anyEngine: 'All engines', - engineGroupLabels: { - trt: 'TRTLLM family', - sglang: 'SGLang/ATOM family', - vllm: 'vLLM family', - } as Record, - engineGroupFallback: (engineGroup: string) => `${engineGroup.toUpperCase()} family`, - primaryHeading: (precision: string) => `${precision} · PRIMARY @${OVERVIEW_PRIMARY_TIER}`, - secondaryHeading: (precision: string) => `${precision} @${OVERVIEW_PRIMARY_TIER}`, - secondaryCoverage: (precision: string, list: string) => - `${precision} coverage: ${list} measured; insufficient comparable results.`, - highHeading: `@${OVERVIEW_HIGH_TIER}`, - leader: 'Leader', - onlyExactResult: 'Only exact result', - highLeader: (hardware: string, value: string) => `${hardware} · ${value}`, - highLeaderSame: 'Same leader', - leaderChange: 'Leader change', - highInsufficient: `No exact read at ${OVERVIEW_HIGH_TIER} tok/s/user`, - highNoPrimaryBaseline: `No ${OVERVIEW_PRIMARY_TIER} tok/s/user baseline`, - evidenceHighDashboard: `Dashboard @${OVERVIEW_HIGH_TIER}`, - evidenceDashboardAria: (model: string, config: string) => - `Open filtered dashboard: ${model} · ${config}`, - evidenceHighAria: (model: string, config: string) => - `Open filtered dashboard @${OVERVIEW_HIGH_TIER} tok/s/user leader: ${model} · ${config}`, + detailLink: 'View details', + detailAria: (modelLabel: string) => `View details: ${modelLabel}`, + dashboardAria: (modelLabel: string, stack: string) => + `Open filtered dashboard: ${modelLabel} · ${stack}`, + delta: (signedPct: string) => `${signedPct} vs B200`, + precisionMismatch: (candidatePrecision: string, baselinePrecision: string) => + `${candidatePrecision.toUpperCase()} vs ${baselinePrecision.toUpperCase()} · no comparable delta`, + versionMismatch: 'Different releases · no comparable delta', + highLeadChanged: (leaderLabel: string) => `At ${OVERVIEW_HIGH_TIER}, ${leaderLabel} leads`, infinityLegend: '∞ = no comparable result', - notRankedReasons: { + missingReasons: (tier: number): Record => ({ standard_decode_only: 'standard decode only', int4_bf16_only: 'INT4/BF16 only', - different_serving_cohort: 'different serving cohort', no_8k1k_data: 'no 8K/1K data', - cannot_reach_at50: `cannot reach @${OVERVIEW_PRIMARY_TIER}`, - no_exact_at50: `no exact @${OVERVIEW_PRIMARY_TIER} result`, - } as Record, - otherPrecisionReason: (precisions: string[]) => - `${precisions.map((precision) => precision.toUpperCase()).join('/')} only`, - joinList: (labels: string[]) => - labels.length <= 1 - ? (labels[0] ?? '') - : `${labels.slice(0, -1).join(', ')} and ${labels.at(-1) ?? ''}`, + cannot_reach_at_tier: `cannot reach @${tier}`, + no_exact_at_tier: `no exact @${tier} result`, + }), methodologyNote: - 'Precisions and incompatible serving cohorts are ranked separately. The default precision maximizes comparable hardware coverage. Not ranked does not mean slower.', + "Each cell shows the platform's best validated speculative-decode serving configuration for that model, labeled with its precision. Deltas against B200 compare only same-precision, same-release results — FP4 is never measured against FP8. Results compare complete serving stacks rather than isolated silicon.", interpolationNote: 'Tier values interpolate each configuration’s official Pareto frontier — no extrapolation.', - cohortNote: - 'Engine families with different speculative-decode acceptance forcing, and aggregated versus disaggregated deployments, rank as separate cohorts — never against each other.', - detailLink: 'View details', - detailAria: (modelLabel: string) => `View details: ${modelLabel}`, }, zh: { title: 'AI 推理总览', - purpose: '对比活跃模型在不同硬件上的已验证服务结果。', - scope: `8K→1K · 单轮 · ${OVERVIEW_PRIMARY_TIER} tok/s/user 下的输出 tok/s/GPU · 仅投机解码 · 按可比覆盖选择 FP4/FP8`, + purpose: '一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。', + scope: (tier: number) => + `8K→1K · 单轮 · 每 GPU 输出 tok/s @${tier} tok/s/用户 · 仅推测解码 · 各平台最佳验证配置`, + tierNavLabel: '服务档位', + tierUnit: 'tok/s/用户', snapshot: (through: string) => `数据库快照截至 ${through}`, + caption: '各活跃模型在关键平台上的最佳验证服务结果,均与 B200 对比。', modelHeader: '模型', - resultsHeader: '排名结果', detailsHeader: '详情', - caption: `各模型与可对比分组在 ${OVERVIEW_PRIMARY_TIER} 与 ${OVERVIEW_HIGH_TIER} tok/s/user 下的已验证输出 tok/s/GPU。`, - cohort: (engine: string, mode: string) => `${engine} · ${mode}`, - aggregated: '聚合部署', - disaggregated: '分离部署', - anyEngine: '全部引擎', - engineGroupLabels: { - trt: 'TRTLLM 系列', - sglang: 'SGLang/ATOM 系列', - vllm: 'vLLM 系列', - } as Record, - engineGroupFallback: (engineGroup: string) => `${engineGroup.toUpperCase()} 系列`, - primaryHeading: (precision: string) => `${precision} · 主排名 @${OVERVIEW_PRIMARY_TIER}`, - secondaryHeading: (precision: string) => `${precision} @${OVERVIEW_PRIMARY_TIER}`, - secondaryCoverage: (precision: string, list: string) => - `${precision} 覆盖:已测量 ${list};可比结果不足。`, - highHeading: `@${OVERVIEW_HIGH_TIER}`, - leader: '领先', - onlyExactResult: '唯一精确读数', - highLeader: (hardware: string, value: string) => `${hardware} · ${value}`, - highLeaderSame: '领先者相同', - leaderChange: '领先者变化', - highInsufficient: `${OVERVIEW_HIGH_TIER} tok/s/user 下无精确读数`, - highNoPrimaryBaseline: `无 ${OVERVIEW_PRIMARY_TIER} tok/s/user 基线`, - evidenceHighDashboard: `仪表板 @${OVERVIEW_HIGH_TIER}`, - evidenceDashboardAria: (model: string, config: string) => - `打开筛选后的仪表板:${model} · ${config}`, - evidenceHighAria: (model: string, config: string) => - `打开筛选后的仪表板 @${OVERVIEW_HIGH_TIER} tok/s/user:${model} · ${config}`, + detailLink: '查看详情', + detailAria: (modelLabel: string) => `查看详情:${modelLabel}`, + dashboardAria: (modelLabel: string, stack: string) => + `打开筛选后的仪表板:${modelLabel} · ${stack}`, + delta: (signedPct: string) => `相对 B200 ${signedPct}`, + precisionMismatch: (candidatePrecision: string, baselinePrecision: string) => + `${candidatePrecision.toUpperCase()} 与 ${baselinePrecision.toUpperCase()} · 无可比差值`, + versionMismatch: '版本不同 · 无可比差值', + highLeadChanged: (leaderLabel: string) => `${OVERVIEW_HIGH_TIER} 档由 ${leaderLabel} 领先`, infinityLegend: '∞ = 无可比结果', - notRankedReasons: { + missingReasons: (tier: number): Record => ({ standard_decode_only: '仅标准解码', int4_bf16_only: '仅 INT4/BF16', - different_serving_cohort: '不同服务配置', no_8k1k_data: '无 8K/1K 数据', - cannot_reach_at50: `无法达到 @${OVERVIEW_PRIMARY_TIER}`, - no_exact_at50: `无精确 @${OVERVIEW_PRIMARY_TIER} 结果`, - } as Record, - otherPrecisionReason: (precisions: string[]) => - `仅 ${precisions.map((precision) => precision.toUpperCase()).join('/')}`, - joinList: (labels: string[]) => labels.join('、'), + cannot_reach_at_tier: `无法达到 @${tier}`, + no_exact_at_tier: `无精确 @${tier} 结果`, + }), methodologyNote: - '不同精度和不可直接比较的服务配置将分别排名。默认精度优先覆盖更多可比较硬件。未参与排名不代表性能更低。', + '每格展示该平台在该模型上表现最好的已验证推测解码服务配置,并标注精度。相对 B200 的差值只在同精度、同版本结果之间计算——绝不将 FP4 与 FP8 互比。对比对象是完整服务栈,而非单独的芯片。', interpolationNote: '各档位数据基于各配置官方 Pareto 前沿插值;不进行外推。', - cohortNote: - '投机解码接受率强制方式不同的引擎系列,以及聚合与分离部署,按独立分组排名,彼此之间不作比较。', - detailLink: '查看详情', - detailAria: (modelLabel: string) => `查看详情:${modelLabel}`, }, } as const; @@ -151,19 +102,11 @@ export type OverviewStrings = (typeof OVERVIEW_STRINGS)[OverviewLocale]; interface Formatters { number: Intl.NumberFormat; - signed: Intl.NumberFormat; - date: (date: string) => string; shortDate: (date: string) => string; } export function overviewFormatters(locale: OverviewLocale): Formatters { const tag = locale === 'zh' ? 'zh-CN' : 'en-US'; - const dateFormat = new Intl.DateTimeFormat(tag, { - year: 'numeric', - month: 'short', - day: 'numeric', - timeZone: 'UTC', - }); const shortDateFormat = new Intl.DateTimeFormat(tag, { month: 'short', day: 'numeric', @@ -171,8 +114,6 @@ export function overviewFormatters(locale: OverviewLocale): Formatters { }); return { number: new Intl.NumberFormat(tag, { maximumFractionDigits: 0 }), - signed: new Intl.NumberFormat(tag, { maximumFractionDigits: 0, signDisplay: 'exceptZero' }), - date: (date) => dateFormat.format(new Date(`${date}T00:00:00Z`)), shortDate: (date) => shortDateFormat.format(new Date(`${date}T00:00:00Z`)), }; } @@ -193,448 +134,192 @@ function formatEvidenceDate( : `${from}–${formatters.shortDate(evidenceDate.to)}`; } -function cohortLabel( - group: OverviewComparisonGroup, - strings: OverviewStrings, - showDbModel: boolean, -): string { - const engine = - group.engineGroup === null - ? strings.anyEngine - : (strings.engineGroupLabels[group.engineGroup] ?? - strings.engineGroupFallback(group.engineGroup)); - const mode = - group.deploymentMode === 'disaggregated' ? strings.disaggregated : strings.aggregated; - const label = strings.cohort(engine, mode); - // Point releases of one display model rank in separate cohorts; name the raw - // db model so two same-engine, same-mode cohorts are told apart. - return showDbModel ? `${label} · ${group.dbModel}` : label; -} - -/** Cohorts with at least one exact @50 read — the ones that render ranked chips. */ -function rankedCohorts(groups: readonly OverviewComparisonGroup[]): OverviewComparisonGroup[] { - return groups.filter((group) => group.primaryRanking.state !== 'insufficient_coverage'); +/** The reason copy for a missing side, keyed to the displayed tier. */ +function missingReasonCopy(member: OverviewHeadlinePairMember, strings: OverviewStrings): string { + const reason = member.missingReason; + return reason === null ? '' : strings.missingReasons(member.read.tier)[reason]; } -/** A cohort's hardware that hold an exact @50 read, leader first then value desc. */ -function rankedStatuses(group: OverviewComparisonGroup): OverviewHardwareStatus[] { - return group.hardwareStatuses - .filter((status) => status.isPrimaryLeader || status.primaryDeltaPercent !== null) - .toSorted((a, b) => (b.primary.value ?? -1) - (a.primary.value ?? -1)); -} - -/** Not-ranked reason copy; `other_precision_only` names the measured precisions. */ -function notRankedReason(entry: OverviewNotRankedEntry, strings: OverviewStrings): string { - return entry.reason === 'other_precision_only' - ? strings.otherPrecisionReason(entry.precisions) - : strings.notRankedReasons[entry.reason]; -} - -/** The exact deployment topology of a ranked configuration, for an aria-label. */ -function evidenceConfigLabel( - config: OverviewConfigResult, - strings: OverviewStrings, - showDbModel: boolean, -): string { - return [ - config.hardwareLabel, - config.precision.toUpperCase(), - config.frameworkLabel, - config.specLabel, - config.disagg ? strings.disaggregated : strings.aggregated, - config.parallelism, - `${config.totalGpu} GPU`, - ...(showDbModel ? [config.dbModel] : []), - ].join(' · '); +/** `candidate/B200 − 1` as a rounded percent with a U+2212 or `+` sign. */ +function signedPercent(formatters: Formatters, percent: number): string { + const rounded = Math.round(percent); + const sign = rounded < 0 ? '−' : '+'; + return `${sign}${formatters.number.format(Math.abs(rounded))}%`; } -const BLOCK_HEADING_CLASS = - 'mb-0.5 text-xs font-semibold uppercase tracking-wider text-muted-foreground'; -const SUBORDINATE_HEADING_CLASS = - 'mb-0.5 text-[11px] font-medium uppercase tracking-wider text-muted-foreground/80'; - /** - * One ranked hardware: its label, its @50 value linking to the filtered - * dashboard, and its standing — `Leader`, its signed delta against the leader, - * or `Only exact result` when it is the cohort's lone exact read. + * The @100 line, rendered only when the @100 leader is a different hardware + * than the displayed tier's leader — the leader being the side with the + * greater exact @100 read. Null on the 100 view itself (data layer). */ -function RankedChip({ - locale, - model, - group, - status, - multiDbModel, - formatters, - strings, -}: { - locale: OverviewLocale; - model: OverviewModelSummary; - group: OverviewComparisonGroup; - status: OverviewHardwareStatus; - multiDbModel: boolean; - formatters: Formatters; - strings: OverviewStrings; -}) { - const { value, config } = status.primary; - const standing = status.isPrimaryLeader - ? group.primaryRanking.state === 'single_measured' - ? strings.onlyExactResult - : strings.leader - : `${formatters.signed.format(status.primaryDeltaPercent as number)}%`; - const date = formatEvidenceDate(formatters, status.primary.evidenceDate); - return ( -

- {status.hardwareLabel} - {value === null ? null : config === null ? ( - - {formatters.number.format(value)} - - ) : ( - - - {formatters.number.format(value)} - - - )} - · {standing} - {date === null ? null : ( - · {date} - )} -

- ); +function highLeadLine( + pair: OverviewHeadlinePairComparison, + strings: OverviewStrings, +): string | null { + if (pair.highLeaderTransition !== 'changed_hardware') return null; + const candidateHigh = pair.candidate.highRead.value; + const baselineHigh = pair.baseline.highRead.value; + if (candidateHigh === null || baselineHigh === null) return null; + const leaderLabel = + candidateHigh > baselineHigh ? pair.candidate.hardwareLabel : pair.baseline.hardwareLabel; + return strings.highLeadChanged(leaderLabel); } -/** A precision's ranked cohorts as stacked chips, under one heading. */ -function RankedSection({ - locale, - model, - cohorts, - heading, - headingClass, - formatters, - strings, -}: { - locale: OverviewLocale; - model: OverviewModelSummary; - cohorts: OverviewComparisonGroup[]; - heading: string; - headingClass: string; - formatters: Formatters; - strings: OverviewStrings; -}) { - if (cohorts.length === 0) return null; - const multiDbModel = new Set(cohorts.map((group) => group.dbModel)).size > 1; - return ( -
-

{heading}

- {cohorts.map((group) => ( -
- {cohorts.length > 1 ? ( -

- {cohortLabel(group, strings, multiDbModel)} -

- ) : null} - {rankedStatuses(group).map((status) => ( - - ))} -
- ))} -
- ); -} +const PAIR_VALUE_LINK_CLASS = + 'inline-flex min-h-11 items-center rounded-sm underline decoration-dotted underline-offset-4 hover:decoration-solid focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50'; -/** The primary precision block: its heading names the precision and the tier. */ -function PrimarySection({ - locale, - model, - cohorts, - formatters, - strings, -}: { - locale: OverviewLocale; - model: OverviewModelSummary; - cohorts: OverviewComparisonGroup[]; - formatters: Formatters; - strings: OverviewStrings; -}) { - const precision = (model.selectedPrecision ?? '').toUpperCase(); +/** A side with no exact read at the displayed tier: `∞` plus its reason, no link/date/percent. */ +function CellMissing({ hardware, reason }: { hardware: string; reason: string }) { return ( - + + + {reason} + ); } /** - * The secondary precision, subordinate to the primary: a ranked block when it - * genuinely adds comparable hardware, otherwise one compact coverage line so the - * precision still surfaces without a second full table. + * One platform's read: its exact displayed-tier value linked to the platform's + * own filtered dashboard, with a precision badge and the evidence date — or + * `∞` with the reason it has no exact read. */ -function SecondarySection({ +function CellValue({ locale, model, + member, formatters, strings, }: { locale: OverviewLocale; model: OverviewModelSummary; + member: OverviewHeadlinePairMember; formatters: Formatters; strings: OverviewStrings; }) { - const secondary = model.secondary; - if (secondary === null) return null; - const precision = secondary.precision.toUpperCase(); - if (secondary.state === 'coverage') { - const list = strings.joinList(secondary.measuredHardware); - if (list === '') return null; - return ( -

{strings.secondaryCoverage(precision, list)}

- ); + const { value, config, evidenceDate } = member.read; + if (member.missingReason !== null || value === null) { + return ; } + // The full deployment stack behind this read; the framework and precision + // stay visible, the spec method rides the link title and aria only. + const stack = + config === null + ? null + : `${member.hardwareLabel} · ${config.frameworkLabel} · ${config.precision.toUpperCase()} · ${config.specLabel}`; return ( - - ); -} - -/** - * The 100 tok/s/user read of one cohort. It is ranked from its own evidence, so - * it names its own leader and says plainly whether that leader changed. - */ -function HighTierRead({ - group, - formatters, - strings, -}: { - group: OverviewComparisonGroup; - formatters: Formatters; - strings: OverviewStrings; -}) { - const leader = group.highRanking.leader; - const value = leader === null ? null : overviewHighValue(leader); - if (leader === null || value === null) { - return {strings.highInsufficient}; - } - // The @100 leader's own read date — from its hardware's high read, so a @100 - // read backed by a different config than the @50 leader shows its own date. - const date = formatEvidenceDate( - formatters, - group.hardwareStatuses.find((status) => status.hardware === leader.hardware)?.high - .evidenceDate ?? null, - ); - return ( - - - {strings.highLeader(leader.hardwareLabel, formatters.number.format(value))} +
+ + {config === null || stack === null ? ( + formatters.number.format(value) + ) : ( + + {formatters.number.format(value)} + + )} - {group.highRanking.state === 'single_measured' ? ( - {strings.onlyExactResult} - ) : group.highLeaderTransition === 'changed_hardware' ? ( - - {strings.leaderChange} - - ) : group.highLeaderTransition === 'no_primary_baseline' ? ( - {strings.highNoPrimaryBaseline} - ) : ( - {strings.highLeaderSame} + {member.precision === null ? null : ( + + {config === null + ? member.precision.toUpperCase() + : `${config.frameworkLabel} · ${config.precision.toUpperCase()}`} + )} - {date === null ? null : ( - · {date} + {evidenceDate === null ? null : ( + + {formatEvidenceDate(formatters, evidenceDate)} + )} - +
); } -/** - * The @100 capability read per comparable cohort. When the @100 leader runs a - * different configuration than the @50 leader, its own filtered dashboard rides - * along so the high-tier read ships with the evidence behind it. - */ -function HighSection({ - locale, - model, - cohorts, - formatters, - strings, -}: { +/** The B200 reference cell: the read alone, no delta against itself. */ +function BaselineCell(props: { locale: OverviewLocale; model: OverviewModelSummary; - cohorts: OverviewComparisonGroup[]; + member: OverviewHeadlinePairMember; formatters: Formatters; strings: OverviewStrings; }) { - const highCohorts = cohorts.filter((group) => group.primaryRanking.state === 'comparable'); - if (highCohorts.length === 0) return null; - const multiDbModel = new Set(highCohorts.map((group) => group.dbModel)).size > 1; return ( -
-

{strings.highHeading}

- {highCohorts.map((group) => { - const primaryLeader = group.primaryRanking.leader; - const highLeader = group.highRanking.leader; - return ( -
- {highCohorts.length > 1 ? ( - - {cohortLabel(group, strings, multiDbModel)} - - ) : null} - - {highLeader !== null && - (primaryLeader === null || highLeader.key !== primaryLeader.key) ? ( - - {strings.evidenceHighDashboard} - - ) : null} -
- ); - })} +
+
); } /** - * Every remaining hardware, accounted for in one line: those with no exact @50 - * read and not ranked by the secondary block, each with its explicit reason. The - * standing note "not ranked does not mean slower" lives once in the methodology. + * One candidate platform's cell: its own read, then the same-precision delta + * against B200, a mismatch note when both sides exist but cannot compare, and + * the @100 leader line only when the leader flips. */ -function NotRankedLine({ - model, - strings, -}: { - model: OverviewModelSummary; - strings: OverviewStrings; -}) { - // Hardware already named by the secondary section — its ranked rows or its - // coverage 'measured' list — is not repeated here; dedup against the secondary's - // measured labels so each hardware surfaces in exactly one place. - const secondaryMeasured = new Set(model.secondary?.measuredHardware); - const entries = model.notRanked.filter((entry) => !secondaryMeasured.has(entry.hardwareLabel)); - if (entries.length === 0) return null; - // ∞ marks missing/unavailable only — it never enters ranking or gap math, so it - // must never render with a percent. The reason rides in title/aria; the page - // footer carries the legend. - return ( -

- {entries.map((entry) => { - const reason = notRankedReason(entry, strings); - return ( - - {entry.hardwareLabel} - - ); - })} -

- ); -} - -/** The full per-model hierarchy, identical on both surfaces. */ -function ModelHierarchy({ +function CandidateCell({ locale, model, + pair, formatters, strings, }: { locale: OverviewLocale; model: OverviewModelSummary; + pair: OverviewHeadlinePairComparison; formatters: Formatters; strings: OverviewStrings; }) { - const cohorts = rankedCohorts(model.comparisonGroups); + const highLine = highLeadLine(pair, strings); return ( -
- + - - - + {pair.directDeltaPercent === null ? null : ( +

+ {strings.delta(signedPercent(formatters, pair.directDeltaPercent))} +

+ )} + {pair.deltaUnavailableReason === null ? null : ( +

+ {pair.deltaUnavailableReason === 'precision_mismatch' + ? strings.precisionMismatch( + pair.candidate.precision ?? '', + pair.baseline.precision ?? '', + ) + : strings.versionMismatch} +

+ )} + {highLine === null ? null :

{highLine}

}
); } -/** - * The model's name and — for a coverage-only model that shows no ranked chips — - * its freshness. A ranked model dates each result on its own chip instead, so - * the model-level date is only the fallback for a model with nothing to rank. - */ -function ModelIdentity({ - model, - formatters, -}: { - model: OverviewModelSummary; - formatters: Formatters; -}) { - const hasRankedResults = - rankedCohorts(model.comparisonGroups).length > 0 || model.secondary?.state === 'ranked'; - const freshness = - hasRankedResults || model.latestWorkloadDate === null - ? null - : formatters.date(model.latestWorkloadDate); - return ( -
-

{model.modelLabel}

- {freshness ? ( - {freshness} - ) : null} -
- ); +/** The model's display name, shared by both surfaces. */ +function ModelName({ model }: { model: OverviewModelSummary }) { + return

{model.modelLabel}

; } interface SurfaceProps { @@ -644,8 +329,10 @@ interface SurfaceProps { strings: OverviewStrings; } -/** Model column plus one content column carrying the full hierarchy; no scroll. */ +/** Model column, the B200 reference, one column per candidate platform, details. */ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: SurfaceProps) { + const headlinePairs = models[0]?.headlinePairs ?? []; + const baselineHeader = headlinePairs[0]?.baseline.hardwareLabel ?? null; return (
- - + + {baselineHeader === null ? null : } + {headlinePairs.map((pair) => ( + + ))} @@ -663,33 +353,54 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S - + {baselineHeader === null ? null : ( + + )} + {headlinePairs.map((pair) => ( + + ))} - {models.map((model) => ( - - + + {models.map((model) => ( + - + {model.headlinePairs[0] === undefined ? null : ( + + )} + {model.headlinePairs.map((pair) => ( + + ))} - - ))} + ))} +
{strings.caption}
{strings.modelHeader} - {strings.resultsHeader} - + {baselineHeader} + + {pair.candidate.hardwareLabel} + {strings.detailsHeader}
- + - - + + + +
); } -/** Below `xl`: the same hierarchy stacked as cards, always fully visible. */ +/** Below `xl`: the same cells stacked as one card per model, always fully visible. */ export function MobileOverviewList({ models, locale, formatters, strings }: SurfaceProps) { return (
    @@ -716,15 +427,39 @@ export function MobileOverviewList({ models, locale, formatters, strings }: Surf
    - - + +
    + {model.headlinePairs[0] === undefined ? null : ( + <> + + {model.headlinePairs[0].baseline.hardwareLabel} + + + + )} + {model.headlinePairs.map((pair) => ( +
    + + {pair.candidate.hardwareLabel} + + +
    + ))} +
    + {strings.tierNavLabel} +
    + {OVERVIEW_TIERS.map((option) => + option === tier ? ( + + {option} + + ) : ( + + {option} + + ), + )} +
    + {strings.tierUnit} + + ); +} + export function OverviewMethodology({ strings }: { strings: OverviewStrings }) { return (

    {strings.methodologyNote}

    {strings.infinityLegend}

    -

    {strings.cohortNote}

    {strings.interpolationNote}

    ); diff --git a/packages/app/src/lib/overview-data.server.ts b/packages/app/src/lib/overview-data.server.ts index a9d3cafa..0bfa5fe6 100644 --- a/packages/app/src/lib/overview-data.server.ts +++ b/packages/app/src/lib/overview-data.server.ts @@ -4,15 +4,25 @@ import { FIXTURES_MODE } from '@semianalysisai/inferencex-db/connection'; import type { BenchmarkRow } from '@/lib/api'; import { getCachedBenchmarks } from '@/lib/benchmark-data.server'; import { DEFAULT_MODELS } from '@/lib/data-mappings'; -import { assembleOverviewPageData, type OverviewPageData } from '@/lib/overview-data'; +import { + assembleOverviewPageData, + OVERVIEW_PRIMARY_TIER, + type OverviewPageData, + type OverviewTier, +} from '@/lib/overview-data'; import { loadFixture } from '@/lib/test-fixtures'; -export async function getOverviewPageData(): Promise { +export async function getOverviewPageData( + tier: OverviewTier = OVERVIEW_PRIMARY_TIER, +): Promise { // E2E fixtures mode serves a small synthetic rows-by-display-model fixture // through the same assembler the live path uses, so a contract drift breaks // the fixture tests instead of silently stranding the page's data. if (FIXTURES_MODE) { - return assembleOverviewPageData(loadFixture>('overview-rows')); + return assembleOverviewPageData( + loadFixture>('overview-rows'), + tier, + ); } // Fetch rows per db model, concatenated per display model (one display model @@ -25,5 +35,5 @@ export async function getOverviewPageData(): Promise { }), ); - return assembleOverviewPageData(Object.fromEntries(entries)); + return assembleOverviewPageData(Object.fromEntries(entries), tier); } diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 96df49f9..29c49a45 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -3,15 +3,11 @@ import { describe, expect, it } from 'vitest'; import overviewRowsFixture from '../../cypress/fixtures/api/overview-rows.json'; import type { BenchmarkRow } from './api'; -import { getHardwareConfig } from './constants'; import { DEFAULT_MODELS, Model, Precision } from './data-mappings'; -import { overviewConfigIdentityKey } from './overview-config-identity'; import { assembleOverviewPageData, - buildOverviewHardwareOrder, buildOverviewModelSummary, - overviewPrimaryValue, - selectOverviewPrecision, + resolveOverviewTier, type OverviewModelSummary, } from './overview-data'; @@ -78,223 +74,63 @@ function frontierAt( ); } -/** One config's frontier at explicit [interactivity, throughput, date] knots. */ -function datedFrontier( - knots: [number, number, string][], - overrides: Partial = {}, -): BenchmarkRow[] { - return knots.map(([intvty, tput, date], index) => - row({ - conc: index + 1, - metrics: { median_intvty: intvty, output_tput_per_gpu: tput }, - date, - ...overrides, - }), - ); +function headlinePairOf(summary: OverviewModelSummary, id: string) { + return summary.headlinePairs.find((pair) => pair.id === id); } -function statusOf(summary: OverviewModelSummary, hardware: string) { - return summary.hardwareStatuses.find((status) => status.hardware === hardware); -} - -/** - * Leader of the model's first comparable cohort. Leadership only ever exists - * inside a cohort, so every ranking assertion reads it from there — there is no - * model-global winner to compare across engine families or deployment modes. - */ -function primaryLeaderOf(summary: OverviewModelSummary) { - return summary.comparisonGroups[0]?.primaryRanking.leader ?? null; -} - -describe('selectOverviewPrecision', () => { - it('returns null without any FP4/FP8 rows', () => { - expect(selectOverviewPrecision(Model.Qwen3_5, [row({ precision: Precision.BF16 })])).toBeNull(); - expect(selectOverviewPrecision(Model.Qwen3_5, [row({ precision: Precision.INT4 })])).toBeNull(); - }); - - it('ignores standard-decode rows when counting exact-@50 coverage', () => { - // FP8 has the most curves but every one is standard decode; FP4's single - // speculative curve is the only exact-@50 coverage, so it wins outnumbered. - const rows = [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), +describe('overview headline pairs', () => { + it('keeps each side on its own best precision and withholds the delta on a mismatch', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { + hardware: 'mi355x', + precision: Precision.FP4, + }), ...frontier([1100, 900, 700, 500], { - hardware: 'gb200', + hardware: 'mi355x', precision: Precision.FP8, - spec_method: 'none', }), ...frontier([1000, 800, 600, 400], { - hardware: 'gb300', + hardware: 'b200', precision: Precision.FP8, - spec_method: 'none', - }), - ]; - - expect(selectOverviewPrecision(Model.Qwen3_5, rows)).toBe(Precision.FP4); - }); -}); - -describe('buildOverviewModelSummary', () => { - it('keeps Kimi point releases as separate exact results, never one blended frontier', () => { - // kimik2.5 and kimik2.7-code both map to Model.Kimi_K2_5. Otherwise - // identical (same hardware/date/config), a model-blind key would merge them - // into one frontier — kimik2.5's 100 tok/s/user point would make the winner - // reachable there. Exact identity keeps kimik2.7-code's frontier pure. - const summary = buildOverviewModelSummary(Model.Kimi_K2_5, [ - row({ - model: 'kimik2.7-code', - conc: 1, - metrics: { median_intvty: 30, output_tput_per_gpu: 1200 }, - }), - row({ - model: 'kimik2.7-code', - conc: 2, - metrics: { median_intvty: 50, output_tput_per_gpu: 1000 }, }), - row({ - model: 'kimik2.7-code', - conc: 3, - metrics: { median_intvty: 75, output_tput_per_gpu: 800 }, - }), - row({ model: 'kimik2.5', conc: 4, metrics: { median_intvty: 50, output_tput_per_gpu: 400 } }), - row({ - model: 'kimik2.5', - conc: 5, - metrics: { median_intvty: 100, output_tput_per_gpu: 700 }, - }), - ]); - - const leader = primaryLeaderOf(summary); - - expect(leader?.dbModel).toBe('kimik2.7-code'); - expect(overviewPrimaryValue(leader!)).toBe(1000); - expect(leader?.tierValues.find(({ tier }) => tier === 100)).toEqual({ - tier: 100, - value: null, - boundary: 'unreachable', - evidenceDate: null, - }); - }); - - it('splits cohorts by db model so point releases never rank against each other', () => { - // Same hardware, engine group and deployment mode; only the db model - // differs, so the two point releases must land in separate cohorts. - const summary = buildOverviewModelSummary(Model.Kimi_K2_5, [ - ...frontier([1200, 1000, 800, 600], { model: 'kimik2.5' }), - ...frontier([1100, 900, 700, 500], { model: 'kimik2.7-code' }), - ]); - - expect(summary.comparisonGroups.map((group) => group.dbModel).toSorted()).toEqual([ - 'kimik2.5', - 'kimik2.7-code', - ]); - }); - - it('never compares incompatible DeepSeek MTP engine families', () => { - const summary = buildOverviewModelSummary(Model.DeepSeek_V4_Pro, [ - ...frontier([1200, 1000, 800, 600], { framework: 'dynamo-trt' }), - ...frontier([1100, 900, 700, 500], { framework: 'mori-sglang' }), ]); - expect(summary.comparisonGroups.map(({ engineGroup }) => engineGroup).toSorted()).toEqual([ - 'sglang', - 'trt', - ]); - expect(summary.comparisonGroups.every((group) => group.primaryRanking.runnerUp === null)).toBe( - true, - ); + const pair = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(pair?.candidate.read.value).toBe(1000); + expect(pair?.candidate.precision).toBe(Precision.FP4); + expect(pair?.baseline.read.value).toBe(800); + expect(pair?.baseline.precision).toBe(Precision.FP8); + expect(pair?.precision).toBeNull(); + expect(pair?.directDeltaPercent).toBeNull(); + expect(pair?.deltaUnavailableReason).toBe('precision_mismatch'); + expect(pair?.highLeaderTransition).toBeNull(); }); - it('keeps aggregate and disaggregate denominators in separate cohorts', () => { + it('breaks equal exact-read coverage toward FP4', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { disagg: false }), - ...frontier([1800, 1500, 1200, 900], { - hardware: 'gb200', - disagg: true, - is_multinode: true, + ...frontier([1200, 1000, 800, 600], { + hardware: 'mi355x', + precision: Precision.FP4, }), - ]); - expect(summary.comparisonGroups.map(({ deploymentMode }) => deploymentMode).toSorted()).toEqual( - ['aggregated', 'disaggregated'], - ); - }); - - it('counts aggregated GPUs as one pool and disaggregated as prefill + decode', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { num_prefill_gpu: 4, num_decode_gpu: 4 }), - ...frontier([1800, 1500, 1200, 900], { - hardware: 'gb200', - disagg: true, - is_multinode: true, - num_prefill_gpu: 16, - num_decode_gpu: 16, + ...frontier([1100, 900, 700, 500], { + hardware: 'b200', + precision: Precision.FP4, + }), + ...frontier([1000, 800, 600, 400], { + hardware: 'mi355x', + precision: Precision.FP8, + }), + ...frontier([900, 700, 500, 300], { + hardware: 'b200', + precision: Precision.FP8, }), ]); - const leaderFor = (mode: string) => - summary.comparisonGroups.find((group) => group.deploymentMode === mode)?.primaryRanking - .leader; - - expect(leaderFor('aggregated')?.totalGpu).toBe(4); - expect(leaderFor('disaggregated')?.totalGpu).toBe(32); - }); - - it('ranks 50 and 100 independently across hardware with a real gap', () => { - // b200 leads 50 (1000 vs 800) but bottoms out at 100 (300), where gb200's - // shallower curve (400) takes over — two exact reads, so the gap is real. - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 300], { hardware: 'b200' }), - ...frontier([1100, 800, 600, 400], { hardware: 'gb200' }), - ]); - const [group] = summary.comparisonGroups; - const expectedHighConfigKey = overviewConfigIdentityKey(row({ hardware: 'gb200' })); - - expect(group.primaryRanking.state).toBe('comparable'); - expect(group.primaryRanking.leader?.hardware).toBe('b200'); - expect(group.primaryRanking.runnerUp?.hardware).toBe('gb200'); - expect(group.primaryRanking.gapPercent).toBeCloseTo((1000 / 800 - 1) * 100); - expect(group.highRanking.leader?.key).toBe(expectedHighConfigKey); - expect(group.highRanking.leader?.hardware).toBe('gb200'); - expect(group.highLeaderTransition).toBe('changed_hardware'); - }); - - it('picks the best per-hardware config independently at each tier', () => { - // On b200, sglang wins 50 (1000 > 700) while dynamo-trt wins 100 (400 > 200). - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 200], { framework: 'sglang' }), - ...frontier([1100, 700, 500, 400], { framework: 'dynamo-trt' }), - ]); - const [group] = summary.comparisonGroups; - - expect(group.primaryRanking.leader).toMatchObject({ hardware: 'b200', framework: 'sglang' }); - expect(group.highRanking.leader).toMatchObject({ hardware: 'b200', framework: 'dynamo-trt' }); - expect(group.highLeaderTransition).toBe('same_hardware'); - }); - - it('reports no_primary_baseline when 50 has no exact read but 100 does', () => { - // Frontier floor at 60 tok/s/user: tier 50 is clamped_low (no exact read), - // so the primary tier has no leader, while tier 100 sits inside [60, 110] - // and interpolates — a high leader with no primary baseline to compare to. - // FP4 rows so this insufficient-coverage precision is the primary (0-0 exact - // tie → FP4); the transition mechanics under test are precision-agnostic. - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontierAt( - [ - [60, 900], - [75, 800], - [90, 700], - [110, 600], - ], - { precision: Precision.FP4 }, - ), - ]); - const [group] = summary.comparisonGroups; - expect(group.primaryRanking.state).toBe('insufficient_coverage'); - expect(group.highLeaderTransition).toBe('no_primary_baseline'); + expect(headlinePairOf(summary, 'mi355x-vs-b200')?.precision).toBe(Precision.FP4); }); - it('never lets clamped or unreachable reads lead or form a gap at a tier', () => { + it('keeps an FP4 bucket and member boundaries when neither side has an exact read', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - // Frontier floor at 60 tok/s/user → 50 read is clamped_low. ...frontierAt( [ [60, 900], @@ -302,364 +138,310 @@ describe('buildOverviewModelSummary', () => { [80, 700], [90, 600], ], - { hardware: 'b300', precision: Precision.FP4 }, + { hardware: 'mi355x', precision: Precision.FP4 }, ), - // Frontier ceiling at 35 tok/s/user → 50 read is unreachable. ...frontierAt( [ [20, 500], - [25, 450], - [30, 400], - [35, 350], + [30, 450], + [40, 400], + [45, 350], ], { hardware: 'b200', precision: Precision.FP4 }, ), ]); - const [group] = summary.comparisonGroups; - expect(group.primaryRanking.state).toBe('insufficient_coverage'); - expect(group.primaryRanking.gapPercent).toBeNull(); - expect(group.primaryRanking.leader).toBeNull(); - expect(group.hardwareStatuses.find((s) => s.hardware === 'b300')?.primary.boundary).toBe( - 'clamped_low', - ); + const pair = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(pair?.precision).toBeNull(); + expect(pair?.candidate.precision).toBe(Precision.FP4); + expect(pair?.candidate.dbModel).toBe('qwen3.5'); + expect(pair?.baseline.precision).toBe(Precision.FP4); + expect(pair?.candidate.read).toMatchObject({ + value: null, + boundary: 'clamped_low', + config: { hardware: 'mi355x' }, + }); + expect(pair?.baseline.read).toMatchObject({ + value: null, + boundary: 'unreachable', + config: { hardware: 'b200' }, + }); + expect(pair?.candidate.missingReason).toBe('no_exact_at_tier'); + expect(pair?.baseline.missingReason).toBe('cannot_reach_at_tier'); }); - it('lets a lower exact read outrank a higher clamped one', () => { - // b300's clamped 900 is its frontier floor, not a 50 tok/s/user result, so - // gb200's real 500 leads and no gap is claimed against unmeasured coverage. + it('keeps an exact side and marks an unreachable side missing without a delta', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1200, 1000, 800, 600], { + hardware: 'mi355x', + precision: Precision.FP4, + }), ...frontierAt( [ - [60, 900], - [70, 800], - [80, 700], - [90, 600], + [20, 500], + [30, 450], + [40, 400], + [45, 350], ], - { hardware: 'b300' }, + { hardware: 'b200', precision: Precision.FP4 }, ), - ...frontier([700, 500, 400, 300], { hardware: 'gb200' }), ]); - const [group] = summary.comparisonGroups; - expect(group.primaryRanking.leader?.hardware).toBe('gb200'); - expect(group.primaryRanking.state).toBe('single_measured'); - expect(group.primaryRanking.gapPercent).toBeNull(); - expect(group.hardwareStatuses.find((s) => s.hardware === 'b300')?.primary).toMatchObject({ - value: 900, - boundary: 'clamped_low', - }); + const pair = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(pair?.candidate.read).toMatchObject({ value: 1000, boundary: 'interpolated' }); + expect(pair?.baseline.read).toMatchObject({ value: null, boundary: 'unreachable' }); + expect(pair?.baseline.missingReason).toBe('cannot_reach_at_tier'); + expect(pair?.directDeltaPercent).toBeNull(); }); - it('dates a model by its newest workload row, and null when it has none', () => { + it('selects member precisions independently and shares a pair precision only when they match', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { date: '2026-07-01' }), - ...frontier([1100, 900, 700, 500], { hardware: 'gb200', date: '2026-07-15' }), + ...frontier([1200, 1000, 800, 600], { + hardware: 'mi355x', + precision: Precision.FP8, + }), + ...frontier([1100, 900, 700, 500], { + hardware: 'b200', + precision: Precision.FP8, + }), + ...frontier([1300, 1100, 900, 700], { + hardware: 'gb300', + precision: Precision.FP4, + }), + ...frontier([1000, 800, 600, 400], { + hardware: 'b200', + precision: Precision.FP4, + }), ]); - expect(summary.latestWorkloadDate).toBe('2026-07-15'); - expect(buildOverviewModelSummary(Model.Qwen3_5, []).latestWorkloadDate).toBeNull(); + const matched = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(matched?.precision).toBe(Precision.FP8); + expect(matched?.directDeltaPercent).not.toBeNull(); + + const mismatched = headlinePairOf(summary, 'gb300-vs-b200'); + expect(mismatched?.candidate.precision).toBe(Precision.FP4); + expect(mismatched?.baseline.precision).toBe(Precision.FP8); + expect(mismatched?.precision).toBeNull(); + expect(mismatched?.deltaUnavailableReason).toBe('precision_mismatch'); }); -}); -describe('overview precision selection and secondary', () => { - it('makes the wider exact-@50 coverage the primary precision', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), - ...frontier([1100, 900, 700, 500], { hardware: 'gb200', precision: Precision.FP8 }), - ...frontier([1000, 800, 600, 400], { hardware: 'gb300', precision: Precision.FP8 }), + it('shows each release’s own read but never deltas across releases', () => { + const summary = buildOverviewModelSummary(Model.Kimi_K2_5, [ + ...frontier([1200, 1000, 800, 600], { + model: 'kimik2.5', + hardware: 'b200', + precision: Precision.FP4, + date: '2026-07-10', + }), + ...frontier([1100, 900, 700, 500], { + model: 'kimik2.7-code', + hardware: 'mi355x', + precision: Precision.FP4, + date: '2026-07-20', + }), ]); - expect(summary.selectedPrecision).toBe(Precision.FP8); + const pair = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(pair?.candidate.read.config?.dbModel).toBe('kimik2.7-code'); + expect(pair?.baseline.read.config?.dbModel).toBe('kimik2.5'); + expect(pair?.dbModel).toBeNull(); + expect(pair?.directDeltaPercent).toBeNull(); + expect(pair?.deltaUnavailableReason).toBe('version_mismatch'); + expect(pair?.highLeaderTransition).toBeNull(); }); - it('breaks an exact-@50 coverage tie toward FP4', () => { + it('computes signed candidate-relative-to-B200 deltas', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), - ...frontier([1100, 900, 700, 500], { hardware: 'gb200', precision: Precision.FP8 }), + ...frontier([1000, 800, 600, 400], { + hardware: 'mi355x', + precision: Precision.FP4, + }), + ...frontier([1200, 1000, 800, 600], { + hardware: 'b200', + precision: Precision.FP4, + }), + ...frontier([1400, 1200, 1000, 800], { + hardware: 'gb300', + precision: Precision.FP4, + }), ]); - expect(summary.selectedPrecision).toBe(Precision.FP4); + expect(headlinePairOf(summary, 'mi355x-vs-b200')?.directDeltaPercent).toBeCloseTo(-20); + expect(headlinePairOf(summary, 'gb300-vs-b200')?.directDeltaPercent).toBeCloseTo(20); }); - it('ranks each precision within itself, never across FP4 and FP8', () => { + it('reports a hardware leader flip at @100 using independently best configs', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), - ...frontier([1100, 900, 700, 500], { hardware: 'b200', precision: Precision.FP4 }), - ...frontier([1000, 800, 600, 400], { hardware: 'gb200', precision: Precision.FP8 }), + ...frontier([1400, 1200, 800, 400], { + hardware: 'mi355x', + precision: Precision.FP4, + framework: 'sglang', + }), + ...frontier([1000, 700, 650, 600], { + hardware: 'mi355x', + precision: Precision.FP4, + framework: 'dynamo-trt', + }), + ...frontier([1200, 1000, 800, 650], { + hardware: 'b200', + precision: Precision.FP4, + }), ]); - // The FP4 cohort ranks only its own two hardware; the FP8 platform is the - // secondary precision, never a cross-precision delta in the primary cohort. - expect(summary.comparisonGroups[0].hardwareStatuses.map((s) => s.hardware).toSorted()).toEqual([ - 'b200', - 'b300', - ]); - expect(summary.secondary?.precision).toBe(Precision.FP8); + const pair = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(pair?.candidate.read.config?.framework).toBe('sglang'); + expect(pair?.candidate.highRead.config?.framework).toBe('dynamo-trt'); + expect(pair?.highLeaderTransition).toBe('changed_hardware'); }); - it('keeps FP8 visible as secondary coverage when FP4 is primary', () => { + it('distinguishes standard-decode-only and unsupported-precision coverage per member', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), - ...frontier([1100, 900, 700, 500], { hardware: 'b200', precision: Precision.FP4 }), - ...frontier([1000, 800, 600, 400], { hardware: 'h200', precision: Precision.FP8 }), + ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), + row({ hardware: 'mi355x', precision: Precision.FP8, spec_method: 'none' }), + row({ hardware: 'b300', precision: Precision.INT4 }), ]); - expect(summary.selectedPrecision).toBe(Precision.FP4); - expect(summary.secondary?.state).toBe('coverage'); - expect(summary.secondary?.measuredHardware).toContain(getHardwareConfig('h200').label); + expect(headlinePairOf(summary, 'mi355x-vs-b200')?.candidate.missingReason).toBe( + 'standard_decode_only', + ); + expect(headlinePairOf(summary, 'b300-vs-b200')?.candidate.missingReason).toBe('int4_bf16_only'); }); - it('opens secondary rows only when FP8 ranks a pair on hardware FP4 lacks', () => { - const ranked = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), - ...frontier([1150, 950, 750, 550], { hardware: 'b200', precision: Precision.FP4 }), - ...frontier([1100, 900, 700, 500], { hardware: 'gb300', precision: Precision.FP4 }), - ...frontier([1000, 800, 600, 400], { hardware: 'gb200', precision: Precision.FP8 }), - ...frontier([980, 780, 580, 380], { hardware: 'mi355x', precision: Precision.FP8 }), - ]); - // FP4 leads on three hardware; FP8 ranks a comparable pair on two it lacks. - expect(ranked.selectedPrecision).toBe(Precision.FP4); - expect(ranked.secondary?.state).toBe('ranked'); - - const covered = buildOverviewModelSummary(Model.Qwen3_5, [ - ...frontier([1200, 1000, 800, 600], { hardware: 'gb200', precision: Precision.FP4 }), - ...frontier([1150, 950, 750, 550], { hardware: 'mi355x', precision: Precision.FP4 }), - ...frontier([900, 850, 650, 450], { hardware: 'gb200', precision: Precision.FP8 }), - ...frontier([880, 820, 620, 420], { hardware: 'mi355x', precision: Precision.FP8 }), - ]); - // Same FP8 pair, but both hardware already rank in FP4 → no new coverage. - expect(covered.secondary?.state).toBe('coverage'); - }); -}); + it('always returns all four fixed pairs for an empty model', () => { + const summary = buildOverviewModelSummary(Model.Qwen3_5, []); -describe('overview not-ranked reasons', () => { - it('accounts for every page hardware with an exact value or one reason', () => { - const model = [ - ...frontier([1200, 1000, 800, 600], { hardware: 'b300', precision: Precision.FP4 }), - ...frontier([1100, 900, 700, 500], { hardware: 'b200', precision: Precision.FP4 }), - // gb200 measured at the primary precision, but standard decode only. - ...frontier([1000, 800, 600, 400], { - hardware: 'gb200', - precision: Precision.FP4, - spec_method: 'none', - }), - ]; - // mi355x reaches the page order only because another model measured it. - const other = frontier([900, 800, 700, 600], { hardware: 'mi355x', precision: Precision.FP4 }); - const order = buildOverviewHardwareOrder([...model, ...other]); - const summary = buildOverviewModelSummary(Model.Qwen3_5, model, order); - - // b300/b200 rank (exact @50) and are absent; gb200 and mi355x each get one. - expect(summary.notRanked.map((entry) => [entry.hardware, entry.reason])).toEqual([ - ['gb200', 'standard_decode_only'], - ['mi355x', 'no_8k1k_data'], + expect(summary.headlinePairs.map(({ id }) => id)).toEqual([ + 'mi355x-vs-b200', + 'b300-vs-b200', + 'gb200-vs-b200', + 'gb300-vs-b200', ]); - }); - - it('separates a frontier that tops out below 50 from a missing @50 read', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - // b200 frontier ceiling at 45 tok/s/user → @50 unreachable. - ...frontierAt( - [ - [20, 500], - [30, 450], - [40, 400], - [45, 350], - ], - { hardware: 'b200', precision: Precision.FP4 }, - ), - // b300 frontier floor at 60 tok/s/user → @50 clamped_low, not unreachable. - ...frontierAt( - [ - [60, 900], - [70, 800], - [80, 700], - [90, 600], - ], - { hardware: 'b300', precision: Precision.FP4 }, + expect(summary.headlinePairs.every(({ precision }) => precision === null)).toBe(true); + expect( + summary.headlinePairs.every( + ({ candidate, baseline }) => + candidate.missingReason === 'no_8k1k_data' && baseline.missingReason === 'no_8k1k_data', ), - ]); - const reasonOf = (hardware: string) => - summary.notRanked.find((entry) => entry.hardware === hardware)?.reason; - - expect(reasonOf('b200')).toBe('cannot_reach_at50'); - expect(reasonOf('b300')).toBe('no_exact_at50'); + ).toBe(true); }); }); -describe('overview result-level evidence dates', () => { - it('dates each read from its own config frontier, never a sibling', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...datedFrontier( - [ - [40, 1100, '2026-07-10'], - [60, 900, '2026-07-10'], - ], - { hardware: 'b300', precision: Precision.FP4 }, - ), - ...datedFrontier( - [ - [40, 900, '2026-07-16'], - [60, 700, '2026-07-16'], - ], - { hardware: 'b200', precision: Precision.FP4 }, - ), - ]); - - expect(statusOf(summary, 'b300')?.primary.evidenceDate).toEqual({ - from: '2026-07-10', - to: '2026-07-10', - }); - expect(statusOf(summary, 'b200')?.primary.evidenceDate).toEqual({ - from: '2026-07-16', - to: '2026-07-16', - }); +describe('tier-parameterized overview', () => { + it('resolves the tier query value and falls back to 50', () => { + expect(resolveOverviewTier('100')).toBe(100); + expect(resolveOverviewTier(['75', '30'])).toBe(75); + expect(resolveOverviewTier('40')).toBe(50); + expect(resolveOverviewTier('')).toBe(50); + expect(resolveOverviewTier(undefined)).toBe(50); }); - it('collapses a same-day bracket and spans a cross-day one', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...datedFrontier( - [ - [40, 1100, '2026-07-10'], - [60, 900, '2026-07-10'], - ], - { hardware: 'b300', precision: Precision.FP4 }, - ), - ...datedFrontier( - [ - [40, 900, '2026-06-24'], - [60, 700, '2026-07-04'], - ], - { hardware: 'b200', precision: Precision.FP4 }, - ), - ]); - - expect(statusOf(summary, 'b300')?.primary.evidenceDate).toEqual({ - from: '2026-07-10', - to: '2026-07-10', - }); - expect(statusOf(summary, 'b200')?.primary.evidenceDate).toEqual({ - from: '2026-06-24', - to: '2026-07-04', - }); + it('stamps the displayed tier on the page and defaults to 50, down to empty models', () => { + expect(assembleOverviewPageData({}).tier).toBe(50); + const page = assembleOverviewPageData({}, 75); + expect(page.tier).toBe(75); + expect(page.models[0]?.headlinePairs[0]?.candidate.read.tier).toBe(75); }); - it('ignores a hidden slower config even when it is newer', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...datedFrontier( - [ - [40, 1100, '2026-07-10'], - [60, 900, '2026-07-10'], - ], - { hardware: 'b300', precision: Precision.FP4, framework: 'sglang' }, - ), - // Slower same-hardware config, newer runs — never backs the visible read. - ...datedFrontier( - [ - [40, 700, '2026-07-20'], - [60, 500, '2026-07-20'], + it('reads every pair member and its delta at the requested tier', () => { + const page = assembleOverviewPageData( + { + [Model.Qwen3_5]: [ + ...frontier([1000, 800, 600, 400], { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), ], - { hardware: 'b300', precision: Precision.FP4, framework: 'dynamo-trt' }, - ), - ]); + }, + 100, + ); - expect(statusOf(summary, 'b300')?.primary.evidenceDate).toEqual({ - from: '2026-07-10', - to: '2026-07-10', - }); + const pair = headlinePairOf( + page.models.find((m) => m.model === Model.Qwen3_5)!, + 'mi355x-vs-b200', + ); + expect(pair?.candidate.read).toMatchObject({ tier: 100, value: 400 }); + expect(pair?.baseline.read).toMatchObject({ tier: 100, value: 600 }); + expect(pair?.directDeltaPercent).toBeCloseTo((400 / 600 - 1) * 100); }); - it('dates @50 and @100 from their own bracketing knots', () => { - const summary = buildOverviewModelSummary(Model.Qwen3_5, [ - ...datedFrontier( + it('turns an unreachable @50 side into an exact read on the 30 view', () => { + const rows = [ + ...frontier([1200, 1000, 800, 600], { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontierAt( [ - [40, 1200, '2026-07-01'], - [60, 1000, '2026-07-01'], - [90, 700, '2026-07-15'], - [110, 500, '2026-07-15'], + [20, 500], + [30, 450], + [40, 400], + [45, 350], ], - { hardware: 'b300', precision: Precision.FP4 }, + { hardware: 'b200', precision: Precision.FP4 }, ), - ]); - const status = statusOf(summary, 'b300'); + ]; + + const at50 = headlinePairOf( + assembleOverviewPageData({ [Model.Qwen3_5]: rows }).models.find( + (m) => m.model === Model.Qwen3_5, + )!, + 'mi355x-vs-b200', + ); + expect(at50?.baseline.missingReason).toBe('cannot_reach_at_tier'); - expect(status?.primary.evidenceDate).toEqual({ from: '2026-07-01', to: '2026-07-01' }); - expect(status?.high.evidenceDate).toEqual({ from: '2026-07-15', to: '2026-07-15' }); + const at30 = headlinePairOf( + assembleOverviewPageData({ [Model.Qwen3_5]: rows }, 30).models.find( + (m) => m.model === Model.Qwen3_5, + )!, + 'mi355x-vs-b200', + ); + expect(at30?.baseline.read).toMatchObject({ tier: 30, value: 450 }); + expect(at30?.baseline.missingReason).toBeNull(); + expect(at30?.directDeltaPercent).toBeCloseTo((1200 / 450 - 1) * 100); }); -}); -describe('overview hardware coverage', () => { - it('holds hardware order fixed against input order and performance changes', () => { - const expected = ['gb300', 'gb200', 'b300', 'b200', 'mi355x']; - const build = (throughputs: number[], hardware: string[]) => { - const rows = hardware.flatMap((hw, index) => - frontier( - [ - throughputs[index] + 200, - throughputs[index], - throughputs[index] - 200, - throughputs[index] - 400, + it('re-selects each platform’s best bucket at the displayed tier', () => { + // FP4 wins @50 (1000 > 900) but tops out low; FP8 wins @100 (700 > 400). + const page = (tier?: 30 | 50 | 75 | 100) => + assembleOverviewPageData( + { + [Model.Qwen3_5]: [ + ...frontier([1200, 1000, 800, 400], { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontier([1100, 900, 850, 700], { hardware: 'mi355x', precision: Precision.FP8 }), + ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP8 }), ], - { hardware: hw }, - ), - ); - return buildOverviewModelSummary(Model.Qwen3_5, rows, buildOverviewHardwareOrder(rows)); - }; - - // Same throughputs, hardware fed in the opposite order: the fastest platform - // swaps from mi355x to gb300 while the display order must not move. - const mi355xLeads = build([600, 700, 800, 900, 1000], expected); - const gb300Leads = build([600, 700, 800, 900, 1000], [...expected].toReversed()); - - expect(mi355xLeads.hardwareStatuses.map(({ hardware }) => hardware)).toEqual(expected); - expect(gb300Leads.hardwareStatuses.map(({ hardware }) => hardware)).toEqual(expected); - expect(primaryLeaderOf(mi355xLeads)).toMatchObject({ hardware: 'mi355x' }); - expect(primaryLeaderOf(gb300Leads)).toMatchObject({ hardware: 'gb300' }); - expect(gb300Leads.comparisonGroups[0].hardwareStatuses.map(({ hardware }) => hardware)).toEqual( - expected, - ); + }, + tier, + ).models.find((m) => m.model === Model.Qwen3_5)!; + + const at50 = headlinePairOf(page(), 'mi355x-vs-b200'); + expect(at50?.candidate.precision).toBe(Precision.FP4); + expect(at50?.deltaUnavailableReason).toBe('precision_mismatch'); + + const at100 = headlinePairOf(page(100), 'mi355x-vs-b200'); + expect(at100?.candidate.precision).toBe(Precision.FP8); + expect(at100?.candidate.read.value).toBe(700); + expect(at100?.directDeltaPercent).toBeCloseTo((700 / 600 - 1) * 100); }); - it('separates unsupported-precision coverage from hardware this model never ran', () => { - // Kimi shape: FP4 standard decode on B200, INT4 only on H200. H200 and - // MI355X reach the page order only because another model measured them. - const kimi = [ - ...frontier([1200, 1000, 800, 600], { - model: 'kimik2.5', - hardware: 'b200', + it('never reports a leader transition on the 100 view', () => { + const rows = [ + ...frontier([1400, 1200, 800, 400], { + hardware: 'mi355x', precision: Precision.FP4, - spec_method: 'none', + framework: 'sglang', }), - ...frontier([900, 800, 700, 600], { - model: 'kimik2.5', - hardware: 'h200', - precision: Precision.INT4, - spec_method: 'none', + ...frontier([1000, 700, 650, 600], { + hardware: 'mi355x', + precision: Precision.FP4, + framework: 'dynamo-trt', }), + ...frontier([1200, 1000, 800, 650], { hardware: 'b200', precision: Precision.FP4 }), ]; - const otherModel = [ - ...frontier([900, 800, 700, 600], { hardware: 'mi355x', precision: Precision.FP4 }), - ...frontier([700, 600, 500, 400], { hardware: 'h200', precision: Precision.FP8 }), - ]; - const order = buildOverviewHardwareOrder([...kimi, ...otherModel]); - const summary = buildOverviewModelSummary(Model.Kimi_K2_5, kimi, order); - - expect(order.map(({ hardware }) => hardware)).toEqual(['b200', 'mi355x', 'h200']); - expect(summary.hardwareStatuses.map(({ hardware }) => hardware)).toEqual([ - 'b200', - 'mi355x', - 'h200', - ]); - expect(statusOf(summary, 'b200')?.coverage.kind).toBe('standard_only'); - expect(statusOf(summary, 'h200')?.coverage).toMatchObject({ - kind: 'unsupported_precision_only', - availablePrecisions: [Precision.INT4], - }); - expect(statusOf(summary, 'mi355x')?.coverage).toMatchObject({ - kind: 'no_workload_data', - availablePrecisions: [], - }); + + const summaryAt = (tier?: 30 | 50 | 75 | 100) => + assembleOverviewPageData({ [Model.Qwen3_5]: rows }, tier).models.find( + (m) => m.model === Model.Qwen3_5, + )!; + expect(headlinePairOf(summaryAt(), 'mi355x-vs-b200')?.highLeaderTransition).toBe( + 'changed_hardware', + ); + expect(headlinePairOf(summaryAt(100), 'mi355x-vs-b200')?.highLeaderTransition).toBeNull(); }); }); @@ -667,74 +449,67 @@ describe('overview hardware coverage', () => { // changes, the hand-built rows in overview-rows.json must be regenerated, and // this fails loudly instead of the e2e stranding empty data. Every expectation // is derived by running this same assembler over the fixture — never eyeballed — -// so it locks the exact 8.7 states overview.cy.ts renders against. +// so it locks the exact matrix cell states overview.cy.ts renders against. describe('assembleOverviewPageData over the overview-rows fixture', () => { - it('serves every 8.7 precision, coverage and evidence-date state through the real builder', () => { + it('serves every matrix cell state through the real builder', () => { const page = assembleOverviewPageData( overviewRowsFixture as unknown as Record, ); expect(page.models).toHaveLength(DEFAULT_MODELS.size); expect(page.datasetThroughDate).toBe('2026-07-18'); - expect(page.hardwareOrder.map((entry) => entry.hardware)).toEqual([ - 'gb300', - 'gb200', - 'b300', - 'b200', - 'mi355x', - 'h200', - 'mi325x', - 'h100', - ]); + expect(page.tier).toBe(50); + + // DeepSeek: FP4 exacts on B200/B300 carry the only same-precision delta with + // a cross-day evidence range; GB200's own best is FP8, so its value shows + // but the delta is withheld; MI355X and GB300 miss in opposite clamp + // directions. + const deepseek = page.models.find((m) => m.model === Model.DeepSeek_V4_Pro)!; + const dsB300 = headlinePairOf(deepseek, 'b300-vs-b200')!; + expect(dsB300.precision).toBe(Precision.FP4); + expect(dsB300.baseline.read.value).toBeCloseTo(900.219); + expect(dsB300.candidate.read.value).toBeCloseTo(1121.875); + expect(dsB300.candidate.read.evidenceDate).toEqual({ from: '2026-06-24', to: '2026-07-04' }); + expect(dsB300.directDeltaPercent).toBeCloseTo(24.62, 1); + const dsGb200 = headlinePairOf(deepseek, 'gb200-vs-b200')!; + expect(dsGb200.candidate.precision).toBe(Precision.FP8); + expect(dsGb200.candidate.read.value).toBe(600); + expect(dsGb200.deltaUnavailableReason).toBe('precision_mismatch'); + expect(headlinePairOf(deepseek, 'mi355x-vs-b200')?.candidate.missingReason).toBe( + 'no_exact_at_tier', + ); + expect(headlinePairOf(deepseek, 'gb300-vs-b200')?.candidate.missingReason).toBe( + 'cannot_reach_at_tier', + ); - // DeepSeek: FP4 primary ranks B300 over B200; FP8 stays visible as one - // coverage line (single measured hardware, not a second table); every other - // page hardware carries exactly one not-ranked reason, including BOTH clamp - // directions; and the leader's @50 read spans two calendar days — the only - // cross-day evidence range in the fixture, so the e2e can exercise the range - // label the current live dataset never produces. - const deepseek = page.models.find((m) => m.model === Model.DeepSeek_V4_Pro); - expect(deepseek?.selectedPrecision).toBe(Precision.FP4); - const dsCohort = deepseek?.comparisonGroups[0]; - expect(dsCohort?.primaryRanking.state).toBe('comparable'); - expect(dsCohort?.primaryRanking.leader?.hardware).toBe('b300'); - expect(dsCohort?.highRanking.leader?.hardware).toBe('b200'); - expect( - dsCohort?.hardwareStatuses.find((s) => s.hardware === 'b300')?.primary.evidenceDate, - ).toEqual({ from: '2026-06-24', to: '2026-07-04' }); - expect(deepseek?.secondary?.state).toBe('coverage'); - expect(deepseek?.secondary?.precision).toBe(Precision.FP8); - expect(deepseek?.secondary?.measuredHardware).toEqual(['GB200 NVL72']); + // MiniMax: only GB300 has workload rows, so its value stands alone — a + // missing baseline never yields a delta OR a mismatch note. + const minimax = page.models.find((m) => m.model === Model.MiniMax_M3)!; + const mmGb300 = headlinePairOf(minimax, 'gb300-vs-b200')!; + expect(mmGb300.baseline.missingReason).toBe('no_8k1k_data'); + expect(mmGb300.candidate.read.value).toBe(700); + expect(mmGb300.directDeltaPercent).toBeNull(); + expect(mmGb300.deltaUnavailableReason).toBeNull(); + + // Qwen: FP8 exacts on both sides yield the −16% delta and a leader flip at + // 100, while B300's FP4 best withholds its delta against the FP8 baseline. + const qwen = page.models.find((m) => m.model === Model.Qwen3_5)!; + const qwenMi = headlinePairOf(qwen, 'mi355x-vs-b200')!; + expect(qwenMi.precision).toBe(Precision.FP8); + expect(qwenMi.directDeltaPercent).toBeCloseTo(-15.56, 1); + expect(qwenMi.highLeaderTransition).toBe('changed_hardware'); + const qwenB300 = headlinePairOf(qwen, 'b300-vs-b200')!; + expect(qwenB300.candidate.precision).toBe(Precision.FP4); + expect(qwenB300.candidate.read.value).toBeCloseTo(1150.625); + expect(qwenB300.deltaUnavailableReason).toBe('precision_mismatch'); + + // Kimi: no workload rows at all → every member of every pair says so. + const kimi = page.models.find((m) => m.model === Model.Kimi_K2_5)!; expect( - Object.fromEntries( - (deepseek?.notRanked ?? []).map((entry) => [entry.hardware, entry.reason]), + kimi.headlinePairs.every( + ({ candidate, baseline }) => + candidate.missingReason === 'no_8k1k_data' && baseline.missingReason === 'no_8k1k_data', ), - ).toMatchObject({ - gb300: 'cannot_reach_at50', - mi355x: 'no_exact_at50', - h100: 'standard_decode_only', - gb200: 'other_precision_only', - h200: 'no_8k1k_data', - }); - - // MiniMax: the wider exact-@50 FP8 coverage flips the primary precision to - // FP8 (the coverage rule), dropping FP4 to a single-hardware coverage line. - const minimax = page.models.find((m) => m.model === Model.MiniMax_M3); - expect(minimax?.selectedPrecision).toBe(Precision.FP8); - expect(minimax?.comparisonGroups[0]?.primaryRanking.leader?.hardware).toBe('h200'); - expect(minimax?.secondary?.state).toBe('coverage'); - expect(minimax?.secondary?.precision).toBe(Precision.FP4); - expect(minimax?.secondary?.measuredHardware).toEqual(['H200']); - - // Qwen: FP4 primary, and FP8 secondary earns a FULL ranked block — it ranks a - // comparable pair AND adds MI355X, hardware FP4 has no exact @50 read for. - const qwen = page.models.find((m) => m.model === Model.Qwen3_5); - expect(qwen?.selectedPrecision).toBe(Precision.FP4); - expect(qwen?.secondary?.state).toBe('ranked'); - expect(qwen?.secondary?.precision).toBe(Precision.FP8); - const qwenSecondaryCohort = qwen?.secondary?.comparisonGroups[0]; - expect(qwenSecondaryCohort?.primaryRanking.state).toBe('comparable'); - expect(qwenSecondaryCohort?.primaryRanking.leader?.hardware).toBe('b200'); - expect(qwen?.secondary?.measuredHardware).toEqual(['B200', 'MI355X']); + ).toBe(true); }); }); diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 6a4e4d59..17365c7d 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -5,24 +5,24 @@ import { parallelismLabel } from '@/components/inference/utils/parallelism-label import type { BenchmarkRow } from './api'; import { buildAvailabilityHwKey } from './chart-utils'; import { getHardwareConfig, getModelSortIndex } from './constants'; -import { - DEFAULT_MODELS, - getModelExclusion, - getModelLabel, - Precision, - type Model, -} from './data-mappings'; -import { buildExclusion, type Exclusion } from './exclusion'; +import { DEFAULT_MODELS, getModelLabel, Precision, type Model } from './data-mappings'; import { overviewConfigIdentityKey } from './overview-config-identity'; import { computeTcoFeed, type TcoTierBoundary } from './tco-feed'; export const OVERVIEW_WORKLOAD = { isl: 8192, osl: 1024 } as const; export const OVERVIEW_TIERS = [30, 50, 75, 100] as const; -/** Headline service point every platform is ranked at. */ +export type OverviewTier = (typeof OVERVIEW_TIERS)[number]; +/** Default service point; ?tier= re-renders the matrix at any OVERVIEW_TIERS value. */ export const OVERVIEW_PRIMARY_TIER = 50; /** High-interactivity capability read; also drives the high-tier leader transition. */ export const OVERVIEW_HIGH_TIER = 100; +/** The `?tier=` query value as a displayable tier; anything else falls back to 50. */ +export function resolveOverviewTier(raw: string | string[] | undefined): OverviewTier { + const candidate = Number(Array.isArray(raw) ? raw[0] : raw); + return OVERVIEW_TIERS.find((tier) => tier === candidate) ?? OVERVIEW_PRIMARY_TIER; +} + export interface OverviewTierValue { tier: number; value: number | null; @@ -81,179 +81,84 @@ export interface OverviewTierRead { } /** - * Why a hardware is or is not competing in the model's ranking. Coverage is - * derived from the raw workload rows alone, so a hardware never disappears just - * because a filter or a ranking excluded it: + * Why a pair member shows `∞` instead of a value, from most to least + * fundamental gap: * - * - `comparable_spec` — speculative results at the selected precision, ranked; - * - `standard_only` — selected precision measured, but standard decode only; - * - `alternate_precision_only` — measured at the other FP4/FP8 precision; - * - `other_engine_group_only` — speculative, but only in an engine - * comparability group no other hardware here shares; - * - `unsupported_precision_only` — measured only outside FP4/FP8; - * - `no_workload_data` — no 8K/1K single-turn row for this model. + * - `no_8k1k_data` — no 8K/1K single-turn row for this hardware at all; + * - `int4_bf16_only` — measured only outside FP4/FP8; + * - `standard_decode_only` — FP4/FP8 measured, but standard decode only; + * - `cannot_reach_at_tier` — speculative frontier tops out below the + * displayed tier; + * - `no_exact_at_tier` — no exact read at the displayed tier for any other + * reason (an under-swept frontier that starts above the tier). */ -export type OverviewCoverageKind = - | 'comparable_spec' - | 'standard_only' - | 'alternate_precision_only' - | 'other_engine_group_only' - | 'unsupported_precision_only' - | 'no_workload_data'; - -export interface OverviewHardwareCoverage { - hardware: string; - hardwareLabel: string; - kind: OverviewCoverageKind; - availablePrecisions: string[]; -} +export type OverviewMissingReason = + | 'standard_decode_only' + | 'int4_bf16_only' + | 'no_8k1k_data' + | 'cannot_reach_at_tier' + | 'no_exact_at_tier'; -/** One hardware in the page-wide display order, which never encodes performance. */ -export interface OverviewHardwareOrderEntry { - hardware: string; - hardwareLabel: string; -} +export const OVERVIEW_HEADLINE_PAIR_DEFINITIONS = [ + { id: 'mi355x-vs-b200', candidateHardware: 'mi355x', baselineHardware: 'b200' }, + { id: 'b300-vs-b200', candidateHardware: 'b300', baselineHardware: 'b200' }, + { id: 'gb200-vs-b200', candidateHardware: 'gb200', baselineHardware: 'b200' }, + { id: 'gb300-vs-b200', candidateHardware: 'gb300', baselineHardware: 'b200' }, +] as const; -export interface OverviewHardwareStatus { +export type OverviewHeadlinePairId = (typeof OVERVIEW_HEADLINE_PAIR_DEFINITIONS)[number]['id']; + +export interface OverviewHeadlinePairMember { hardware: string; hardwareLabel: string; - coverage: OverviewHardwareCoverage; /** - * Primary-tier read of this hardware's best configuration in the surrounding - * config set — the same read the ranking used, so a row can never display a - * number the ranking did not. Null read when the hardware has no ranked - * configuration; `coverage.kind` says why. + * The precision of this platform's own best qualified speculative read at + * the displayed tier, selected independently per platform (exact first, then + * value, tie → FP4). Null when the platform has no FP4/FP8 speculative + * bucket at all. */ - primary: OverviewTierRead; - /** High-tier read, taken from this hardware's own best config at that tier. */ - high: OverviewTierRead; - isPrimaryLeader: boolean; - /** Signed percentage against the primary leader; null for the leader itself - * and whenever either side is not an exact in-range read. */ - primaryDeltaPercent: number | null; -} - -/** Whether a cohort's throughput denominator is aggregated or disaggregated. */ -export type OverviewDeploymentMode = 'aggregated' | 'disaggregated'; - -/** - * Ranking state at one tier: `comparable` (≥2 exact hardware reads, gap - * computable), `single_measured` (one exact read), or `insufficient_coverage` - * (no exact read — every read is clamped or unreachable). - */ -export type OverviewRankingState = 'comparable' | 'single_measured' | 'insufficient_coverage'; - -export interface OverviewTierRanking { - tier: number; - state: OverviewRankingState; - leader: OverviewConfigResult | null; - runnerUp: OverviewConfigResult | null; - gapPercent: number | null; -} - -/** - * One comparable cohort: exact configs that share an engine comparability group - * (per the model's exclusion policy) and a deployment mode. Tiers are ranked - * independently inside a cohort; configs never compare across cohorts. - */ -export interface OverviewComparisonGroup { - id: string; - dbModel: string; - engineGroup: string | null; - deploymentMode: OverviewDeploymentMode; - hardwareStatuses: OverviewHardwareStatus[]; - primaryRanking: OverviewTierRanking; - highRanking: OverviewTierRanking; + precision: string | null; + /** The dbModel (point release) backing this platform's selected read. */ + dbModel: string | null; + /** Exact read when available; non-exact reads retain their boundary but never a numeric value. */ + read: OverviewTierRead; + /** Best @100 read from this platform's own selected dbModel and precision. */ + highRead: OverviewTierRead; + missingReason: OverviewMissingReason | null; +} + +export interface OverviewHeadlinePairComparison { + id: OverviewHeadlinePairId; + label: string; + /** The precision shared by both displayed reads when the delta is comparable; null otherwise. */ + precision: string | null; + /** The dbModel shared by both displayed reads when the delta is comparable; null otherwise. */ + dbModel: string | null; + candidate: OverviewHeadlinePairMember; + baseline: OverviewHeadlinePairMember; /** - * How the high-tier leader relates to the primary-tier leader: a new hardware - * took the lead (`changed_hardware`), the same hardware held it - * (`same_hardware`), there was no primary leader to compare against - * (`no_primary_baseline`), or the high tier has no leader at all (`null`). + * Candidate relative to B200. Non-null only when both displayed-tier reads + * are exact, share one precision AND one dbModel, and B200 > 0 — a delta + * never mixes FP4 with FP8 or two point releases. */ - highLeaderTransition: 'same_hardware' | 'changed_hardware' | 'no_primary_baseline' | null; -} - -export interface OverviewSecondaryPrecision { - precision: string; - /** 'ranked' → full rows render; 'coverage' → one compact line renders. */ - state: 'ranked' | 'coverage'; - /** Populated when state === 'ranked'; [] otherwise. Built by the SAME pipeline as the primary (cohorts, rankings, transitions) over the secondary precision's rows. */ - comparisonGroups: OverviewComparisonGroup[]; - hardwareStatuses: OverviewHardwareStatus[]; - /** Hardware labels measured at this precision (any decode mode), page order. Always populated. */ - measuredHardware: string[]; -} - -/** - * Why a page hardware carries no visible value for a model. Derived from the - * existing coverage kinds and ranking outcomes, never a parallel truth: - * - * - `standard_decode_only` — from `standard_only`; - * - `other_precision_only` — from `alternate_precision_only` and unranked by - * the secondary block; the renderer prints 'FP8 only'/'FP4 only'; - * - `int4_bf16_only` — from `unsupported_precision_only`; - * - `different_serving_cohort` — from `other_engine_group_only`; - * - `no_8k1k_data` — from `no_workload_data`; - * - `cannot_reach_at50` — comparable spec whose frontier tops out below 50; - * - `no_exact_at50` — comparable spec with no exact @50 read for any other reason. - */ -export type OverviewNotRankedReason = - | 'standard_decode_only' - | 'other_precision_only' - | 'int4_bf16_only' - | 'different_serving_cohort' - | 'no_8k1k_data' - | 'cannot_reach_at50' - | 'no_exact_at50'; - -export interface OverviewNotRankedEntry { - hardware: string; - hardwareLabel: string; - reason: OverviewNotRankedReason; - /** FP4/FP8 precisions this hardware measured, for the 'FP8 only' style copy. */ - precisions: string[]; + directDeltaPercent: number | null; + /** Why two displayed exact reads still carry no delta; null when a delta exists or a side is missing. */ + deltaUnavailableReason: 'precision_mismatch' | 'version_mismatch' | null; + highLeaderTransition: 'same_hardware' | 'changed_hardware' | null; } export interface OverviewModelSummary { model: Model; modelLabel: string; - /** - * Primary precision: whichever of FP4/FP8 covers more unique hardware with an - * exact @50 speculative read; a tie (including 0-0 when FP4/FP8 rows exist) - * goes to FP4. Null only when the model has no FP4/FP8 workload rows at all. - */ - selectedPrecision: string | null; - /** - * Every page hardware's coverage and tier reads for this model, in the - * page-wide order. Leadership is claimed here only when a single cohort - * exists; with several cohorts the claim belongs to `comparisonGroups`. - */ - hardwareStatuses: OverviewHardwareStatus[]; - /** Comparable cohorts, each ranked independently at the primary and high tiers. */ - comparisonGroups: OverviewComparisonGroup[]; - /** The other of FP4/FP8, or null when it has no workload rows. Ranks only - * within itself — never against the primary precision. */ - secondary: OverviewSecondaryPrecision | null; - /** - * One entry per page hardware with no visible value — neither an exact @50 - * read in the primary block nor a ranked read in a `ranked` secondary. Every - * page hardware is therefore accounted for per model: an exact value, or one - * not-ranked reason. - */ - notRanked: OverviewNotRankedEntry[]; - /** - * Newest workload row measured for this model. Now that every visible result - * carries its own `evidenceDate`, this exists ONLY to date a coverage-only - * model with zero visible results. Null when the model has no workload row. - */ - latestWorkloadDate: string | null; - emptyReason: 'no_8k1k_single_turn_data' | 'no_fp4_fp8_data' | null; + /** Fixed candidate-vs-B200 serving-stack comparisons, one per matrix column. */ + headlinePairs: OverviewHeadlinePairComparison[]; } export interface OverviewPageData { models: OverviewModelSummary[]; - hardwareOrder: { hardware: string; hardwareLabel: string }[]; datasetThroughDate: string | null; + /** The service point every headline pair was read at. */ + tier: OverviewTier; } /** Precisions the overview may rank, in preference order. */ @@ -269,25 +174,6 @@ function overviewWorkloadRows(rows: readonly BenchmarkRow[]): BenchmarkRow[] { ); } -/** - * Page-wide hardware display order: every base hardware with FP4/FP8 coverage - * at the overview workload, in hardware-registry order. Built before precision - * selection and before any ranking, so neither a precision fallback nor a - * measurement change can reorder or drop a column. - */ -export function buildOverviewHardwareOrder( - rows: readonly BenchmarkRow[], -): OverviewHardwareOrderEntry[] { - const hardware = new Set( - overviewWorkloadRows(rows) - .filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)) - .map((row) => row.hardware), - ); - return [...hardware] - .toSorted((a, b) => getModelSortIndex(a) - getModelSortIndex(b) || a.localeCompare(b)) - .map((entry) => ({ hardware: entry, hardwareLabel: getHardwareConfig(entry).label })); -} - /** * Newest raw workload row across the page. Deliberately not derived from the * retained winners: a precision or engine the page never ranks still dates the @@ -300,14 +186,6 @@ export function overviewDatasetThroughDate(rows: readonly BenchmarkRow[]): strin ); } -/** Newest date among a model's own workload rows; null when it has none. */ -function latestWorkloadDateOf(workloadRows: readonly BenchmarkRow[]): string | null { - return workloadRows.reduce( - (latest, row) => (latest === null || row.date > latest ? row.date : latest), - null, - ); -} - /** * Speculative configs for one precision, grouped by exact deployment identity. * Standard-decode rows never enter the pool, so a precision is ranked on its @@ -335,56 +213,6 @@ function buildPrecisionConfigs( return configs; } -/** Unique hardware whose best @50 read is exact (in-range) among `configs`. */ -function exactPrimaryHardware(configs: readonly OverviewConfigResult[]): Set { - return new Set( - [...readsByHardwareAtTier(configs, OVERVIEW_PRIMARY_TIER).values()] - .filter(isExactTierRead) - .map((read) => read.config.hardware), - ); -} - -/** Whichever precision covers more exact-@50 hardware; a tie (incl. 0-0) → FP4. */ -function selectPrimaryFrom( - configsByPrecision: ReadonlyMap, -): string { - const exactCount = (precision: string) => - exactPrimaryHardware(configsByPrecision.get(precision) ?? []).size; - return exactCount(Precision.FP8) > exactCount(Precision.FP4) ? Precision.FP8 : Precision.FP4; -} - -/** - * Primary precision by exact-@50 hardware coverage: FP4 and FP8 are built and - * counted independently, the wider coverage wins, and a tie (including 0-0 when - * FP4/FP8 rows exist) goes to FP4. Null only when no FP4/FP8 workload row exists - * — coverage volume of one vendor's curves can shift which precision is primary, - * but never which precisions are ranked. - */ -export function selectOverviewPrecision( - model: Model, - workloadRows: readonly BenchmarkRow[], -): string | null { - if (!workloadRows.some((row) => OVERVIEW_PRECISIONS.includes(row.precision))) return null; - const configsByPrecision = new Map( - OVERVIEW_PRECISIONS.map( - (precision) => [precision, buildPrecisionConfigs(model, workloadRows, precision)] as const, - ), - ); - return selectPrimaryFrom(configsByPrecision); -} - -function tierValueAt(config: OverviewConfigResult, tier: number): number | null { - return config.tierValues.find((value) => value.tier === tier)?.value ?? null; -} - -export function overviewPrimaryValue(config: OverviewConfigResult): number | null { - return tierValueAt(config, OVERVIEW_PRIMARY_TIER); -} - -export function overviewHighValue(config: OverviewConfigResult): number | null { - return tierValueAt(config, OVERVIEW_HIGH_TIER); -} - function readConfigAtTier(config: OverviewConfigResult, tier: number): OverviewTierRead { const tierValue = config.tierValues.find((value) => value.tier === tier); return { @@ -420,8 +248,7 @@ function compareTierReads(a: ConfigTierRead, b: ConfigTierRead): number { /** * One read per hardware at `tier`: its best exact in-range read, or — only when * the hardware has no exact read at all — its best out-of-range read, kept so - * the hardware still surfaces as a coverage gap. Ranking and hardware rows both - * consume this, so a row can never display a number the ranking did not use. + * the hardware still surfaces as a coverage gap rather than disappearing. */ function readsByHardwareAtTier( configs: readonly OverviewConfigResult[], @@ -445,199 +272,188 @@ function nullTierRead(tier: number): OverviewTierRead { return { tier, value: null, boundary: null, evidenceDate: null, config: null }; } -/** - * Classify one hardware's coverage from its raw workload rows. `comparable` - * lists the hardware sharing an engine comparability group with at least one - * other hardware; a platform whose only speculative results sit in an engine - * family nobody else here uses is reported as coverage rather than a competitor. - */ -function coverageKindFor( - hardwareRows: readonly BenchmarkRow[], - selectedPrecision: string | null, - hardware: string, - comparable: ReadonlySet, -): OverviewCoverageKind { - if (hardwareRows.length === 0) return 'no_workload_data'; - const candidates = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); - if (candidates.length === 0) return 'unsupported_precision_only'; - const selected = candidates.filter((row) => row.precision === selectedPrecision); - if (selected.length === 0) return 'alternate_precision_only'; - if (!selected.some((row) => row.spec_method !== 'none')) return 'standard_only'; - return comparable.size === 0 || comparable.has(hardware) - ? 'comparable_spec' - : 'other_engine_group_only'; -} - -/** - * One status row per entry of `coverage`, which already carries the stable - * hardware order. `configs` scopes the tier reads and the leader: the whole - * model's ranked set at model level, one cohort's set inside a comparison group. - */ -function buildHardwareStatuses( - coverage: readonly OverviewHardwareCoverage[], - configs: readonly OverviewConfigResult[], - withLeader: boolean, -): OverviewHardwareStatus[] { - const primaryReads = readsByHardwareAtTier(configs, OVERVIEW_PRIMARY_TIER); - const highReads = readsByHardwareAtTier(configs, OVERVIEW_HIGH_TIER); - const leader = withLeader - ? [...primaryReads.values()].filter(isExactTierRead).toSorted(compareTierReads)[0] - : undefined; - - return coverage.map((hardwareCoverage) => { - const { hardware, hardwareLabel } = hardwareCoverage; - const primary = primaryReads.get(hardware) ?? nullTierRead(OVERVIEW_PRIMARY_TIER); - const isPrimaryLeader = leader !== undefined && primary.config?.key === leader.config.key; - return { - hardware, - hardwareLabel, - coverage: hardwareCoverage, - primary, - high: highReads.get(hardware) ?? nullTierRead(OVERVIEW_HIGH_TIER), - isPrimaryLeader, - primaryDeltaPercent: - leader !== undefined && !isPrimaryLeader && isExactTierRead(primary) && leader.value > 0 - ? (primary.value / leader.value - 1) * 100 - : null, - }; - }); -} - -/** - * Rank one comparable cohort at a single tier. Every configuration is read at - * `tier`, each hardware contributes only its best exact read, and the surviving - * hardware are ordered by that read — so each tier is ranked on its own - * evidence and a 50 winner is never carried into the 100 ranking. - */ -function rankAtTier(configs: readonly OverviewConfigResult[], tier: number): OverviewTierRanking { - const hardwareLeaders = [...readsByHardwareAtTier(configs, tier).values()] - .filter(isExactTierRead) - .toSorted(compareTierReads); - - const [leader, runnerUp] = hardwareLeaders; - return { - tier, - state: - hardwareLeaders.length >= 2 - ? 'comparable' - : hardwareLeaders.length === 1 - ? 'single_measured' - : 'insufficient_coverage', - leader: leader?.config ?? null, - runnerUp: runnerUp?.config ?? null, - gapPercent: - leader && runnerUp && runnerUp.value > 0 ? (leader.value / runnerUp.value - 1) * 100 : null, - }; +function nonComparableAsMissing( + read: OverviewTierRead | undefined, + tier: number, +): OverviewTierRead { + if (read === undefined) return nullTierRead(tier); + return isExactTierRead(read) ? read : { ...read, value: null, evidenceDate: null }; } -interface OverviewCohort { +interface OverviewHeadlinePairBucket { + precision: string; dbModel: string; - engineGroup: string | null; - deploymentMode: OverviewDeploymentMode; configs: OverviewConfigResult[]; + tierReads: Map; + exactCount: number; + newestEvidence: string; } -/** - * How a cohort's high-tier leader relates to its primary-tier leader. Compared - * on hardware, not config identity: a same-hardware engine or precision swap - * between tiers is not a hardware change, and a high tier with no primary - * baseline to compare against says so rather than implying a change. - */ -function computeHighLeaderTransition( - primaryRanking: OverviewTierRanking, - highRanking: OverviewTierRanking, -): OverviewComparisonGroup['highLeaderTransition'] { - if (highRanking.leader === null) return null; - if (primaryRanking.leader === null) return 'no_primary_baseline'; - return highRanking.leader.hardware === primaryRanking.leader.hardware - ? 'same_hardware' - : 'changed_hardware'; -} +function buildHeadlinePairBuckets( + configsByPrecision: ReadonlyMap, + hardware: ReadonlySet, + tier: OverviewTier, +): OverviewHeadlinePairBucket[] { + const buckets: OverviewHeadlinePairBucket[] = []; + for (const precision of OVERVIEW_PRECISIONS) { + const byDbModel = new Map(); + for (const config of configsByPrecision.get(precision) ?? []) { + if (!hardware.has(config.hardware)) continue; + const configs = byDbModel.get(config.dbModel); + if (configs) configs.push(config); + else byDbModel.set(config.dbModel, [config]); + } -function buildComparisonGroup( - id: string, - cohort: OverviewCohort, - coverage: readonly OverviewHardwareCoverage[], -): OverviewComparisonGroup { - const cohortHardware = new Set(cohort.configs.map(({ hardware }) => hardware)); - const primaryRanking = rankAtTier(cohort.configs, OVERVIEW_PRIMARY_TIER); - const highRanking = rankAtTier(cohort.configs, OVERVIEW_HIGH_TIER); - return { - id, - dbModel: cohort.dbModel, - engineGroup: cohort.engineGroup, - deploymentMode: cohort.deploymentMode, - hardwareStatuses: buildHardwareStatuses( - coverage.filter(({ hardware }) => cohortHardware.has(hardware)), - cohort.configs, - true, - ), - primaryRanking, - highRanking, - highLeaderTransition: computeHighLeaderTransition(primaryRanking, highRanking), - }; + for (const [dbModel, configs] of byDbModel) { + const tierReads = readsByHardwareAtTier(configs, tier); + const exactReads = [...tierReads.values()].filter(isExactTierRead); + buckets.push({ + precision, + dbModel, + configs, + tierReads, + exactCount: exactReads.length, + newestEvidence: exactReads.reduce( + (latest, read) => + read.evidenceDate !== null && read.evidenceDate.to > latest + ? read.evidenceDate.to + : latest, + configs.reduce( + (latest, config) => (config.latestDate > latest ? config.latestDate : latest), + '', + ), + ), + }); + } + } + return buckets; } -/** - * Partition configurations into cohorts that may actually be compared: only - * configs sharing a db model (one display model spans several point releases, - * which must never rank against each other), an engine comparability group (the - * model's exclusion policy — DeepSeek MTP acceptance forcing differs per engine - * family) and a throughput denominator (aggregated vs disaggregated) are ranked - * against each other. - */ -function buildCohorts( - exclusion: Exclusion, - configs: readonly OverviewConfigResult[], -): Map { - const cohorts = new Map(); - for (const config of configs) { - const engineGroup = exclusion.groupOf(config.hwKey); - const deploymentMode: OverviewDeploymentMode = config.disagg ? 'disaggregated' : 'aggregated'; - const id = `${config.dbModel}|${engineGroup ?? 'any'}|${deploymentMode}`; - const cohort = cohorts.get(id); - if (cohort) cohort.configs.push(config); - else - cohorts.set(id, { dbModel: config.dbModel, engineGroup, deploymentMode, configs: [config] }); +function missingReasonForHeadlineMember( + workloadRows: readonly BenchmarkRow[], + hardware: string, + read: OverviewTierRead, +): OverviewMissingReason | null { + if (isExactTierRead(read)) return null; + const hardwareRows = workloadRows.filter((row) => row.hardware === hardware); + if (hardwareRows.length === 0) return 'no_8k1k_data'; + const supportedRows = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); + if (supportedRows.length === 0) return 'int4_bf16_only'; + if (!supportedRows.some((row) => row.spec_method !== 'none')) return 'standard_decode_only'; + return read.boundary === 'unreachable' ? 'cannot_reach_at_tier' : 'no_exact_at_tier'; +} + +function headlineLeaderTransition( + candidatePrimary: OverviewTierRead, + baselinePrimary: OverviewTierRead, + candidateHigh: OverviewTierRead, + baselineHigh: OverviewTierRead, +): OverviewHeadlinePairComparison['highLeaderTransition'] { + if ( + !isExactTierRead(candidatePrimary) || + !isExactTierRead(baselinePrimary) || + !isExactTierRead(candidateHigh) || + !isExactTierRead(baselineHigh) || + candidatePrimary.value === baselinePrimary.value || + candidateHigh.value === baselineHigh.value + ) { + return null; } - return cohorts; + const primaryLeaderIsCandidate = candidatePrimary.value > baselinePrimary.value; + const highLeaderIsCandidate = candidateHigh.value > baselineHigh.value; + return primaryLeaderIsCandidate === highLeaderIsCandidate ? 'same_hardware' : 'changed_hardware'; } -/** - * Hardware sharing an engine comparability group with at least one other - * platform. Partitioned by engine group alone, never by the ranking cohorts: - * cohorts also split on deployment mode, so a platform measured only - * disaggregated would look isolated while running the very engine family its - * peers run — and `other_engine_group_only` would then assert a reason that is - * false. - */ -function comparableHardware( - exclusion: Exclusion, - configs: readonly OverviewConfigResult[], -): Set { - const byEngineGroup = new Map>(); - for (const config of configs) { - const group = exclusion.groupOf(config.hwKey); - const groupHardware = byEngineGroup.get(group); - if (groupHardware) groupHardware.add(config.hardware); - else byEngineGroup.set(group, new Set([config.hardware])); - } - return new Set( - [...byEngineGroup.values()] - .filter((groupHardware) => groupHardware.size > 1) - .flatMap((groupHardware) => [...groupHardware]), +function buildHeadlinePairs( + model: Model, + workloadRows: readonly BenchmarkRow[], + tier: OverviewTier, +): OverviewHeadlinePairComparison[] { + // FP4 and FP8 configs are built independently and never blend within a read. + const configsByPrecision = new Map( + OVERVIEW_PRECISIONS.map( + (precision) => [precision, buildPrecisionConfigs(model, workloadRows, precision)] as const, + ), ); -} + // Each platform selects its own best qualified bucket (dbModel × precision): + // exact at the displayed tier first, then value, tie → FP4, newest evidence, + // lexical dbModel. One bucket per member keeps point releases from blending + // within a read. + const buildMember = (memberHardware: string): OverviewHeadlinePairMember => { + const best = buildHeadlinePairBuckets(configsByPrecision, new Set([memberHardware]), tier) + .map((memberBucket) => ({ + bucket: memberBucket, + read: nonComparableAsMissing(memberBucket.tierReads.get(memberHardware), tier), + })) + .toSorted( + (a, b) => + Number(isExactTierRead(b.read)) - Number(isExactTierRead(a.read)) || + (b.read.value ?? -1) - (a.read.value ?? -1) || + OVERVIEW_PRECISIONS.indexOf(a.bucket.precision) - + OVERVIEW_PRECISIONS.indexOf(b.bucket.precision) || + b.bucket.newestEvidence.localeCompare(a.bucket.newestEvidence) || + a.bucket.dbModel.localeCompare(b.bucket.dbModel), + )[0]; + const primary = best?.read ?? nullTierRead(tier); + const high = nonComparableAsMissing( + best === undefined + ? undefined + : readsByHardwareAtTier(best.bucket.configs, OVERVIEW_HIGH_TIER).get(memberHardware), + OVERVIEW_HIGH_TIER, + ); + return { + hardware: memberHardware, + hardwareLabel: getHardwareConfig(memberHardware, model).label, + precision: best?.bucket.precision ?? null, + dbModel: best?.bucket.dbModel ?? null, + read: primary, + highRead: high, + missingReason: missingReasonForHeadlineMember(workloadRows, memberHardware, primary), + }; + }; -function buildComparisonGroups( - cohorts: ReadonlyMap, - coverage: readonly OverviewHardwareCoverage[], -): OverviewComparisonGroup[] { - const leadValue = (group: OverviewComparisonGroup): number => - group.primaryRanking.leader ? (overviewPrimaryValue(group.primaryRanking.leader) ?? -1) : -1; - return [...cohorts.entries()] - .map(([id, cohort]) => buildComparisonGroup(id, cohort, coverage)) - .toSorted((a, b) => leadValue(b) - leadValue(a) || a.id.localeCompare(b.id)); + return OVERVIEW_HEADLINE_PAIR_DEFINITIONS.map((definition) => { + const candidate = buildMember(definition.candidateHardware); + const baseline = buildMember(definition.baselineHardware); + const bothExact = isExactTierRead(candidate.read) && isExactTierRead(baseline.read); + const comparable = + bothExact && + candidate.precision === baseline.precision && + candidate.dbModel === baseline.dbModel; + return { + id: definition.id, + label: `${candidate.hardwareLabel} vs ${baseline.hardwareLabel}`, + precision: comparable ? candidate.precision : null, + dbModel: comparable ? candidate.dbModel : null, + candidate, + baseline, + directDeltaPercent: + comparable && + candidate.read.value !== null && + baseline.read.value !== null && + baseline.read.value > 0 + ? (candidate.read.value / baseline.read.value - 1) * 100 + : null, + deltaUnavailableReason: bothExact + ? candidate.precision === baseline.precision + ? candidate.dbModel === baseline.dbModel + ? null + : 'version_mismatch' + : 'precision_mismatch' + : null, + // On the 100 view the displayed reads ARE the high reads, so a + // transition line would restate the cell values. + highLeaderTransition: + comparable && tier !== OVERVIEW_HIGH_TIER + ? headlineLeaderTransition( + candidate.read, + baseline.read, + candidate.highRead, + baseline.highRead, + ) + : null, + }; + }); } function buildConfigResult( @@ -706,264 +522,33 @@ function buildConfigResult( }; } -/** - * Coverage for every hardware in `order`, derived from raw workload rows only. - * Hardware the model never measured stays in the list as `no_workload_data`, - * so a platform is never silently absent from one model's row. - */ -function buildCoverage( - order: readonly OverviewHardwareOrderEntry[], - workloadRows: readonly BenchmarkRow[], - selectedPrecision: string | null, - comparable: ReadonlySet, -): OverviewHardwareCoverage[] { - const rowsByHardware = new Map(); - for (const row of workloadRows) { - const hardwareRows = rowsByHardware.get(row.hardware); - if (hardwareRows) hardwareRows.push(row); - else rowsByHardware.set(row.hardware, [row]); - } - - return order.map(({ hardware, hardwareLabel }) => { - const hardwareRows = rowsByHardware.get(hardware) ?? []; - return { - hardware, - hardwareLabel, - kind: coverageKindFor(hardwareRows, selectedPrecision, hardware, comparable), - availablePrecisions: [...new Set(hardwareRows.map((row) => row.precision))].toSorted(), - }; - }); -} - -interface OverviewPrecisionBlock { - comparisonGroups: OverviewComparisonGroup[]; - hardwareStatuses: OverviewHardwareStatus[]; -} - -/** - * Rank one precision's speculative configs: cohorts, per-tier rankings and the - * page-wide hardware statuses. The primary and secondary precisions are both - * built through here, so neither can rank against the other. - */ -function buildPrecisionBlock( - model: Model, - precision: string, - configs: readonly OverviewConfigResult[], - hardwareOrder: readonly OverviewHardwareOrderEntry[], - workloadRows: readonly BenchmarkRow[], -): OverviewPrecisionBlock { - const exclusion = buildExclusion(getModelExclusion(model)); - const cohorts = buildCohorts(exclusion, configs); - const comparable = comparableHardware(exclusion, configs); - const coverage = buildCoverage(hardwareOrder, workloadRows, precision, comparable); - const comparisonGroups = buildComparisonGroups(cohorts, coverage); - // Leadership is a claim against a comparable cohort. Across several cohorts - // there is no model-global leader, so the model row asserts none. - return { - comparisonGroups, - hardwareStatuses: buildHardwareStatuses(coverage, configs, comparisonGroups.length <= 1), - }; -} - -/** - * The other of FP4/FP8. `ranked` (full rows) iff it both ranks a comparable - * cohort AND covers hardware the primary precision lacks; otherwise `coverage` - * (one compact line, ranked rows suppressed). `measuredHardware` always lists - * every hardware measured at this precision so it never disappears from the page. - */ -function buildSecondaryPrecision( - model: Model, - precision: string, - configs: readonly OverviewConfigResult[], - primaryExact: ReadonlySet, - hardwareOrder: readonly OverviewHardwareOrderEntry[], - workloadRows: readonly BenchmarkRow[], -): OverviewSecondaryPrecision | null { - const measuredSet = new Set( - workloadRows.filter((row) => row.precision === precision).map((row) => row.hardware), - ); - if (measuredSet.size === 0) return null; - const measuredHardware = hardwareOrder - .filter(({ hardware }) => measuredSet.has(hardware)) - .map(({ hardwareLabel }) => hardwareLabel); - - const block = buildPrecisionBlock(model, precision, configs, hardwareOrder, workloadRows); - const addsCoverage = [...exactPrimaryHardware(configs)].some((hw) => !primaryExact.has(hw)); - const hasComparable = block.comparisonGroups.some( - (group) => group.primaryRanking.state === 'comparable', - ); - const ranked = hasComparable && addsCoverage; - return { - precision, - state: ranked ? 'ranked' : 'coverage', - comparisonGroups: ranked ? block.comparisonGroups : [], - hardwareStatuses: ranked ? block.hardwareStatuses : [], - measuredHardware, - }; -} - -/** Map one non-ranked hardware's coverage and read to its not-ranked reason. */ -function notRankedReasonFor(status: OverviewHardwareStatus): OverviewNotRankedReason { - switch (status.coverage.kind) { - case 'standard_only': { - return 'standard_decode_only'; - } - case 'alternate_precision_only': { - return 'other_precision_only'; - } - case 'unsupported_precision_only': { - return 'int4_bf16_only'; - } - case 'other_engine_group_only': { - return 'different_serving_cohort'; - } - case 'no_workload_data': { - return 'no_8k1k_data'; - } - case 'comparable_spec': { - // Comparable-spec hardware reaches here only without an exact @50 read. - // Classify by INTERACTIVITY direction, not the boundary name: 'unreachable' - // = tier 50 above the frontier's max, i.e. even the fastest measured point - // is < 50 tok/s/user (the config is too slow, ever) → cannot_reach_at50. - // 'clamped_low' = tier below the frontier's min, i.e. every measured point - // is > 50 tok/s/user — an under-swept sweep gap more concurrency would - // cross, NOT incapability → no_exact_at50. - return status.primary.boundary === 'unreachable' ? 'cannot_reach_at50' : 'no_exact_at50'; - } - } -} - -/** - * One not-ranked entry per page hardware with no visible value — no exact @50 - * read in the primary block and no ranked read in a `ranked` secondary. Reads - * the same coverage/status machinery the blocks use, so it is a derived view - * rather than a parallel source of truth. - */ -function buildNotRanked( - primaryStatuses: readonly OverviewHardwareStatus[], - secondary: OverviewSecondaryPrecision | null, -): OverviewNotRankedEntry[] { - const secondaryRanked = - secondary?.state === 'ranked' - ? new Set( - secondary.hardwareStatuses - .filter((status) => isExactTierRead(status.primary)) - .map((status) => status.hardware), - ) - : new Set(); - - const entries: OverviewNotRankedEntry[] = []; - for (const status of primaryStatuses) { - if (isExactTierRead(status.primary) || secondaryRanked.has(status.hardware)) continue; - entries.push({ - hardware: status.hardware, - hardwareLabel: status.hardwareLabel, - reason: notRankedReasonFor(status), - precisions: status.coverage.availablePrecisions.filter((p) => - OVERVIEW_PRECISIONS.includes(p), - ), - }); - } - return entries; -} - -function emptyModelSummary( - model: Model, - emptyReason: NonNullable, - order: readonly OverviewHardwareOrderEntry[], - workloadRows: readonly BenchmarkRow[], -): OverviewModelSummary { - const hardwareStatuses = buildHardwareStatuses( - buildCoverage(order, workloadRows, null, new Set()), - [], - false, - ); - return { - model, - modelLabel: getModelLabel(model), - selectedPrecision: null, - hardwareStatuses, - comparisonGroups: [], - secondary: null, - notRanked: buildNotRanked(hardwareStatuses, null), - latestWorkloadDate: latestWorkloadDateOf(workloadRows), - emptyReason, - }; -} - export function buildOverviewModelSummary( model: Model, rows: BenchmarkRow[], - hardwareOrder: readonly OverviewHardwareOrderEntry[] = buildOverviewHardwareOrder(rows), + tier: OverviewTier = OVERVIEW_PRIMARY_TIER, ): OverviewModelSummary { - const workloadRows = overviewWorkloadRows(rows); - if (workloadRows.length === 0) { - return emptyModelSummary(model, 'no_8k1k_single_turn_data', hardwareOrder, workloadRows); - } - - if (!workloadRows.some((row) => OVERVIEW_PRECISIONS.includes(row.precision))) { - return emptyModelSummary(model, 'no_fp4_fp8_data', hardwareOrder, workloadRows); - } - - // FP4 and FP8 speculative configs, built once and reused for selection, the - // primary block and the secondary block — the two precisions never blend. - const configsByPrecision = new Map( - OVERVIEW_PRECISIONS.map( - (precision) => [precision, buildPrecisionConfigs(model, workloadRows, precision)] as const, - ), - ); - const selectedPrecision = selectPrimaryFrom(configsByPrecision); - const secondaryPrecision = OVERVIEW_PRECISIONS.find((p) => p !== selectedPrecision)!; - const primaryConfigs = configsByPrecision.get(selectedPrecision)!; - - const primary = buildPrecisionBlock( - model, - selectedPrecision, - primaryConfigs, - hardwareOrder, - workloadRows, - ); - const secondary = buildSecondaryPrecision( - model, - secondaryPrecision, - configsByPrecision.get(secondaryPrecision)!, - exactPrimaryHardware(primaryConfigs), - hardwareOrder, - workloadRows, - ); - return { model, modelLabel: getModelLabel(model), - selectedPrecision, - hardwareStatuses: primary.hardwareStatuses, - comparisonGroups: primary.comparisonGroups, - secondary, - notRanked: buildNotRanked(primary.hardwareStatuses, secondary), - latestWorkloadDate: latestWorkloadDateOf(workloadRows), - emptyReason: null, + headlinePairs: buildHeadlinePairs(model, overviewWorkloadRows(rows), tier), }; } /** * Assemble the whole page from rows already grouped by DISPLAY model. Iterating * DEFAULT_MODELS fixes the output order and renders every active model — a model - * with no rows falls through to its coverage-only states, exactly as the live - * path does. The page-wide hardware order and dataset date come from the union - * of every model's rows. The live server path and the e2e fixture path feed this - * same function; only the row source differs. + * with no rows still gets its four pairs, every member carrying a missing + * reason. The live server path and the e2e fixture path feed this same + * function; only the row source differs. */ export function assembleOverviewPageData( rowsByModel: Record, + tier: OverviewTier = OVERVIEW_PRIMARY_TIER, ): OverviewPageData { const perModel = [...DEFAULT_MODELS].map((model) => ({ model, rows: rowsByModel[model] ?? [] })); - const allRows = perModel.flatMap(({ rows }) => rows); - const hardwareOrder = buildOverviewHardwareOrder(allRows); return { - models: perModel.map(({ model, rows }) => - buildOverviewModelSummary(model, rows, hardwareOrder), - ), - hardwareOrder, - datasetThroughDate: overviewDatasetThroughDate(allRows), + models: perModel.map(({ model, rows }) => buildOverviewModelSummary(model, rows, tier)), + datasetThroughDate: overviewDatasetThroughDate(perModel.flatMap(({ rows }) => rows)), + tier, }; } diff --git a/packages/app/src/lib/overview-links.test.ts b/packages/app/src/lib/overview-links.test.ts index 0f6c4f35..c509a140 100644 --- a/packages/app/src/lib/overview-links.test.ts +++ b/packages/app/src/lib/overview-links.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { Model, Precision } from './data-mappings'; import type { OverviewConfigResult, OverviewModelSummary } from './overview-data'; -import { buildOverviewDashboardHref } from './overview-links'; +import { buildOverviewDashboardHref, detailHref } from './overview-links'; const RUN_URL = 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/26714221123'; @@ -43,13 +43,7 @@ function summary(overrides: Partial = {}): OverviewModelSu return { model: Model.Qwen3_5, modelLabel: 'Qwen 3.5', - selectedPrecision: Precision.FP4, - hardwareStatuses: [], - comparisonGroups: [], - secondary: null, - notRanked: [], - latestWorkloadDate: null, - emptyReason: null, + headlinePairs: [], ...overrides, }; } @@ -98,3 +92,11 @@ describe('buildOverviewDashboardHref', () => { ); }); }); + +describe('detailHref', () => { + it('keeps the model drilldown precision-neutral because headline pairs may differ', () => { + expect(detailHref('en', summary())).toBe( + '/inference?g_model=Qwen-3.5-397B-A17B&i_seq=8k%2F1k&i_optimal=1', + ); + }); +}); diff --git a/packages/app/src/lib/overview-links.ts b/packages/app/src/lib/overview-links.ts index 8e9deb5e..132def11 100644 --- a/packages/app/src/lib/overview-links.ts +++ b/packages/app/src/lib/overview-links.ts @@ -6,7 +6,12 @@ */ import { runIdFromRunUrl } from './known-issues'; -import type { OverviewConfigResult, OverviewModelSummary } from './overview-data'; +import { + OVERVIEW_PRIMARY_TIER, + type OverviewConfigResult, + type OverviewModelSummary, + type OverviewTier, +} from './overview-data'; import type { UrlStateParams } from './url-state'; /** The single-turn 8K-in/1K-out workload every overview link filters to. */ @@ -75,15 +80,24 @@ export function buildOverviewDashboardHref( } /** - * Model-level dashboard view: the workload and precision this page ranked, with - * no configuration pinned. Cohort-level evidence links narrow it further. + * Model-level dashboard view: precision-neutral, because the two headline pairs + * may select different precisions. Result-level evidence links narrow further. */ export function detailHref(locale: 'en' | 'zh', model: OverviewModelSummary): string { const query = new URLSearchParams({ g_model: model.model, i_seq: OVERVIEW_WORKLOAD_SEQ, - ...(model.selectedPrecision ? { i_prec: model.selectedPrecision } : {}), i_optimal: '1', }); return `${inferenceRoute(locale)}?${query}`; } + +/** + * The overview itself at another service point. The default tier keeps the bare + * canonical URL; other tiers ride a plain `?tier=` query the server re-renders + * from, so any view is a copyable link. + */ +export function overviewTierHref(locale: 'en' | 'zh', tier: OverviewTier): string { + const base = locale === 'zh' ? '/zh/overview' : '/overview'; + return tier === OVERVIEW_PRIMARY_TIER ? base : `${base}?tier=${tier}`; +} diff --git a/packages/app/src/lib/tco-feed.test.ts b/packages/app/src/lib/tco-feed.test.ts index 4775c4d9..d6cf6203 100644 --- a/packages/app/src/lib/tco-feed.test.ts +++ b/packages/app/src/lib/tco-feed.test.ts @@ -180,6 +180,34 @@ describe('computeTcoFeed — frontier reads', () => { expect(rows[0].oldest_frontier_date).toBe('2026-04-25'); }); + it("uses only an exact interior frontier knot's date as evidence", () => { + const rows = computeTcoFeed( + [ + makeRow({ itl: 1 / 20, otput: 1000, date: '2026-04-25' }), + makeRow({ itl: 1 / 50, otput: 400, date: '2026-05-20' }), + makeRow({ itl: 1 / 100, otput: 100, date: '2026-07-12' }), + ], + WORKLOAD_8K1K, + [50], + ); + + expect(rows[0].evidence_date).toEqual({ from: '2026-05-20', to: '2026-05-20' }); + }); + + it('uses both bracketing frontier-knot dates as evidence between knots', () => { + const rows = computeTcoFeed( + [ + makeRow({ itl: 1 / 20, otput: 1000, date: '2026-04-25' }), + makeRow({ itl: 1 / 50, otput: 400, date: '2026-05-20' }), + makeRow({ itl: 1 / 100, otput: 100, date: '2026-07-12' }), + ], + WORKLOAD_8K1K, + [75], + ); + + expect(rows[0].evidence_date).toEqual({ from: '2026-05-20', to: '2026-07-12' }); + }); + it('groups by workload and hardware, sorted by hardware key', () => { const rows = computeTcoFeed( [ diff --git a/packages/app/src/lib/tco-feed.ts b/packages/app/src/lib/tco-feed.ts index 9041b130..083c37d6 100644 --- a/packages/app/src/lib/tco-feed.ts +++ b/packages/app/src/lib/tco-feed.ts @@ -221,6 +221,7 @@ function bracketKnots(frontier: FrontierPoint[], tier: number): [FrontierPoint, if (tier >= frontier[last].interactivity) return [frontier[last], frontier[last]]; let lo = 0; for (let i = 0; i < last; i++) { + if (frontier[i].interactivity === tier) return [frontier[i], frontier[i]]; if (frontier[i].interactivity <= tier) lo = i; } return [frontier[lo], frontier[lo + 1]]; From 517e2cecf5877cc647329d8bb04db646f2585db1 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 19:01:49 -0700 Subject: [PATCH 04/13] refactor(overview): drop dead dashboard-link helper and unreachable date guard --- .../overview/overview-detail-link.tsx | 49 ++----------------- .../overview/overview-scorecard.tsx | 7 ++- 2 files changed, 7 insertions(+), 49 deletions(-) diff --git a/packages/app/src/components/overview/overview-detail-link.tsx b/packages/app/src/components/overview/overview-detail-link.tsx index f7b23298..2e1e44cb 100644 --- a/packages/app/src/components/overview/overview-detail-link.tsx +++ b/packages/app/src/components/overview/overview-detail-link.tsx @@ -5,14 +5,12 @@ import type { ReactNode } from 'react'; import { track } from '@/lib/analytics'; import { cn } from '@/lib/utils'; -import type { OverviewConfigResult, OverviewModelSummary } from '@/lib/overview-data'; -import { buildOverviewDashboardHref } from '@/lib/overview-links'; /** - * Model-level dashboard link. A plain anchor for the same reason the evidence - * links below are: the dashboard reads its filters from a snapshot `url-state.ts` - * takes at module evaluation, so a client-side `` navigation would land on - * an unfiltered dashboard while the label promises the model's own view. + * Model-level dashboard link. A plain anchor, not a ``: the dashboard + * reads its filters from a snapshot `url-state.ts` takes at module evaluation, + * so a client-side navigation would land on an unfiltered dashboard while the + * label promises the model's own view. */ export function OverviewDetailLink({ href, @@ -45,42 +43,3 @@ export function OverviewDetailLink({ ); } - -const EVIDENCE_LINK_CLASS = - 'group inline-flex min-h-11 w-fit items-center gap-1 whitespace-nowrap rounded-sm text-xs text-muted-foreground underline decoration-dotted underline-offset-4 transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50 motion-reduce:transition-none'; - -/** - * One evidence link: the /inference dashboard pre-filtered to a ranked - * configuration. The visible text stays short so the compact rows scan; the - * full deployment topology rides in `ariaLabel`, which the caller also makes - * distinct per cohort — so no two evidence links share the accessible name - * "Open filtered dashboard". - * - * A plain anchor, not a ``: the dashboard reads its share-link params - * from a snapshot `url-state.ts` takes at module evaluation, so a client-side - * navigation would arrive with every filter dropped — silently unfiltered while - * the accessible name promises filtered evidence. - */ -export function OverviewDashboardLink({ - locale, - model, - config, - ariaLabel, - children, -}: { - locale: 'en' | 'zh'; - model: OverviewModelSummary; - config: OverviewConfigResult; - ariaLabel: string; - children: ReactNode; -}) { - return ( - - {children} - - ); -} diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index b5d09806..a6adb504 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -121,13 +121,12 @@ export function overviewFormatters(locale: OverviewLocale): Formatters { /** * A result's evidence dates, localized: a single day (`Jul 6` / `7月6日`) when * both backing frontier points share a date, else an en-dash range - * (`Jun 24–Jul 4`). Null when the value has no backing date. + * (`Jun 24–Jul 4`). The caller renders nothing when there is no backing date. */ function formatEvidenceDate( formatters: Formatters, - evidenceDate: { from: string; to: string } | null, -): string | null { - if (evidenceDate === null) return null; + evidenceDate: { from: string; to: string }, +): string { const from = formatters.shortDate(evidenceDate.from); return evidenceDate.from === evidenceDate.to ? from From c87d8e8ed6974abdba229f529929bfddf763a635 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 23:11:38 -0700 Subject: [PATCH 05/13] refactor(overview): trim unused config fields; 44px minecraft audio targets --- .../minecraft/minecraft-toggles.tsx | 2 +- packages/app/src/lib/overview-data.ts | 37 ------------------- packages/app/src/lib/overview-links.test.ts | 9 ----- 3 files changed, 1 insertion(+), 47 deletions(-) diff --git a/packages/app/src/components/minecraft/minecraft-toggles.tsx b/packages/app/src/components/minecraft/minecraft-toggles.tsx index ccea82b6..073a0bff 100644 --- a/packages/app/src/components/minecraft/minecraft-toggles.tsx +++ b/packages/app/src/components/minecraft/minecraft-toggles.tsx @@ -6,7 +6,7 @@ import { track } from '@/lib/analytics'; import { cn } from '@/lib/utils'; const toggleClasses = cn( - 'inline-flex items-center justify-center rounded-md p-2', + 'inline-flex items-center justify-center rounded-md size-11', 'text-muted-foreground hover:text-foreground hover:bg-accent', 'transition-colors duration-200', 'focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2', diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 17365c7d..1e1f9563 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -1,7 +1,5 @@ import { resolveFrameworkPartLabel } from '@semianalysisai/inferencex-constants'; -import { parallelismLabel } from '@/components/inference/utils/parallelism-label'; - import type { BenchmarkRow } from './api'; import { buildAvailabilityHwKey } from './chart-utils'; import { getHardwareConfig, getModelSortIndex } from './constants'; @@ -47,7 +45,6 @@ export interface OverviewConfigResult { key: string; dbModel: string; hardware: string; - hardwareLabel: string; hwKey: string; framework: string; frameworkLabel: string; @@ -55,19 +52,9 @@ export interface OverviewConfigResult { specLabel: string; disagg: boolean; precision: string; - offloadMode: string; - isMultinode: boolean; - numPrefillGpu: number; - numDecodeGpu: number; - /** Physical GPUs backing this config: one shared pool when aggregated, prefill - * plus decode when disaggregated. */ - totalGpu: number; - parallelism: string; - image: string | null; sourceRunUrls: string[]; tierValues: OverviewTierValue[]; latestDate: string; - oldestFrontierDate: string; } /** One hardware's frontier read at a single tier, with its backing config. */ @@ -467,7 +454,6 @@ function buildConfigResult( const first = rows[0]; const { hardware, framework, spec_method: specMethod, disagg } = first; - const latestRow = rows.reduce((a, b) => (b.date > a.date ? b : a)); const sourceRunUrls = [ ...new Set(rows.flatMap((row) => (row.run_url === null ? [] : [row.run_url]))), ].toSorted(); @@ -475,7 +461,6 @@ function buildConfigResult( key, dbModel: first.model, hardware, - hardwareLabel: getHardwareConfig(hardware, model).label, hwKey: buildAvailabilityHwKey(hardware, framework, specMethod, disagg), framework, frameworkLabel: resolveFrameworkPartLabel(model, framework), @@ -483,27 +468,6 @@ function buildConfigResult( specLabel: resolveFrameworkPartLabel(model, specMethod), disagg, precision, - offloadMode: first.offload_mode ?? 'off', - isMultinode: first.is_multinode, - numPrefillGpu: first.num_prefill_gpu, - numDecodeGpu: first.num_decode_gpu, - totalGpu: disagg ? first.num_prefill_gpu + first.num_decode_gpu : first.num_decode_gpu, - parallelism: parallelismLabel({ - tp: first.decode_tp, - ep: first.decode_ep, - dpAttention: first.decode_dp_attention, - disagg: first.disagg, - isMultinode: first.is_multinode, - prefillTp: first.prefill_tp, - prefillEp: first.prefill_ep, - prefillDpAttention: first.prefill_dp_attention, - prefillNumWorkers: first.prefill_num_workers, - decodeTp: first.decode_tp, - decodeEp: first.decode_ep, - decodeDpAttention: first.decode_dp_attention, - decodeNumWorkers: first.decode_num_workers, - }), - image: latestRow.image, sourceRunUrls, tierValues: feed.map((row) => { const value = @@ -518,7 +482,6 @@ function buildConfigResult( }; }), latestDate: feed[0].latest_date, - oldestFrontierDate: feed[0].oldest_frontier_date, }; } diff --git a/packages/app/src/lib/overview-links.test.ts b/packages/app/src/lib/overview-links.test.ts index c509a140..71406dd6 100644 --- a/packages/app/src/lib/overview-links.test.ts +++ b/packages/app/src/lib/overview-links.test.ts @@ -16,7 +16,6 @@ function config(overrides: Partial = {}): OverviewConfigRe key: 'qwen3.5|b200|sglang|mtp|agg|fp4', dbModel: 'qwen3.5', hardware: 'b200', - hardwareLabel: 'B200', hwKey: 'b200_sglang_mtp', framework: 'sglang', frameworkLabel: 'SGLang', @@ -24,17 +23,9 @@ function config(overrides: Partial = {}): OverviewConfigRe specLabel: 'MTP', disagg: false, precision: Precision.FP4, - offloadMode: 'off', - isMultinode: false, - numPrefillGpu: 8, - numDecodeGpu: 8, - totalGpu: 8, - parallelism: 'TP8', - image: 'inferencex/sglang:v0.5.0', sourceRunUrls: [RUN_URL], tierValues: [{ tier: 50, value: 1000, boundary: 'interpolated', evidenceDate: null }], latestDate: '2026-07-18', - oldestFrontierDate: '2026-07-11', ...overrides, }; } From 73d843e1cae8ed1e2756ba0785660eb856f3f239 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 23:18:43 -0700 Subject: [PATCH 06/13] refactor(overview): drop unread bucket exactCount --- packages/app/src/lib/overview-data.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 1e1f9563..8155d686 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -272,7 +272,6 @@ interface OverviewHeadlinePairBucket { dbModel: string; configs: OverviewConfigResult[]; tierReads: Map; - exactCount: number; newestEvidence: string; } @@ -299,7 +298,6 @@ function buildHeadlinePairBuckets( dbModel, configs, tierReads, - exactCount: exactReads.length, newestEvidence: exactReads.reduce( (latest, read) => read.evidenceDate !== null && read.evidenceDate.to > latest From 4f38f6b50b2ea46795eabf898d0360f5a7a7e2e3 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 23:29:27 -0700 Subject: [PATCH 07/13] fix(header): keep query state when switching language --- packages/app/cypress/e2e/overview.cy.ts | 7 +++++++ packages/app/src/components/header/header.tsx | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 89fb5039..fc63f3be 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -251,6 +251,13 @@ describe('Overview page', () => { cy.get('body') .invoke('text') .should('not.match', /∞\s*%/); + + cy.visit('/overview?tier=100'); + cy.contains('Output tok/s/GPU @100 tok/s/user').should('exist'); + cy.get('[data-testid="language-toggle"]').click(); + cy.location('pathname').should('eq', '/zh/overview'); + cy.location('search').should('eq', '?tier=100'); + cy.contains('每 GPU 输出 tok/s @100 tok/s/用户').should('exist'); }); it('uses the same cell semantics on mobile and fits both 390px and 320px widths', () => { diff --git a/packages/app/src/components/header/header.tsx b/packages/app/src/components/header/header.tsx index 09926859..e029e445 100644 --- a/packages/app/src/components/header/header.tsx +++ b/packages/app/src/components/header/header.tsx @@ -75,7 +75,13 @@ function isActive(pathname: string, href: string): boolean { } /** EN ↔ 中文 switcher; maps the current page to its sibling in the other language. */ -function LanguageToggle({ pathname }: { pathname: string }) { +function LanguageToggle({ + pathname, + router, +}: { + pathname: string; + router: ReturnType; +}) { const isZh = isZhPathname(pathname); const target = switchLocalePath(pathname); return ( @@ -84,7 +90,11 @@ function LanguageToggle({ pathname }: { pathname: string }) { data-testid="language-toggle" hrefLang={isZh ? 'en' : 'zh-CN'} className="inline-flex items-center min-h-11 px-2 rounded-md text-sm font-medium text-muted-foreground hover:text-foreground hover:bg-muted transition-colors whitespace-nowrap" - onClick={() => track('header_language_toggled', { to: isZh ? 'en' : 'zh' })} + onClick={(event) => { + track('header_language_toggled', { to: isZh ? 'en' : 'zh' }); + // The sibling keeps shareable query state (e.g. the overview's ?tier=). + navigateInApp(event, router, target + window.location.search); + }} > {isZh ? 'EN' : '中文'} @@ -191,7 +201,7 @@ export const Header = ({ starCount }: { starCount?: number | null }) => { - + {/* Minecraft-only audio toggles ride alongside GitHub stars below the same `sm` cutoff: in minecraft mode the pixel font widens the bar, and these two extra buttons are what push a 320px header past its From 729987537d2074b82f808a800768b600f97cdbce Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Wed, 22 Jul 2026 23:39:18 -0700 Subject: [PATCH 08/13] fix(overview): cannot-reach reason requires every speculative bucket unreachable --- packages/app/src/lib/overview-data.test.ts | 37 +++++++++++++++++ packages/app/src/lib/overview-data.ts | 46 ++++++++++++++-------- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 29c49a45..61ba6c7f 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -294,6 +294,43 @@ describe('overview headline pairs', () => { expect(pair?.highLeaderTransition).toBe('changed_hardware'); }); + it('claims cannot-reach only when every speculative bucket is unreachable', () => { + const unreachable: [number, number][] = [ + [20, 500], + [30, 450], + [40, 400], + [45, 350], + ]; + const underSwept: [number, number][] = [ + [60, 900], + [70, 800], + [80, 700], + [90, 600], + ]; + const baseline = frontier([1200, 1000, 800, 600], { + hardware: 'b200', + precision: Precision.FP4, + }); + + const mixed = buildOverviewModelSummary(Model.Qwen3_5, [ + ...baseline, + ...frontierAt(unreachable, { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontierAt(underSwept, { hardware: 'mi355x', precision: Precision.FP8 }), + ]); + expect(headlinePairOf(mixed, 'mi355x-vs-b200')?.candidate.missingReason).toBe( + 'no_exact_at_tier', + ); + + const allUnreachable = buildOverviewModelSummary(Model.Qwen3_5, [ + ...baseline, + ...frontierAt(unreachable, { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontierAt(unreachable, { hardware: 'mi355x', precision: Precision.FP8 }), + ]); + expect(headlinePairOf(allUnreachable, 'mi355x-vs-b200')?.candidate.missingReason).toBe( + 'cannot_reach_at_tier', + ); + }); + it('distinguishes standard-decode-only and unsupported-precision coverage per member', () => { const summary = buildOverviewModelSummary(Model.Qwen3_5, [ ...frontier([1200, 1000, 800, 600], { hardware: 'b200', precision: Precision.FP4 }), diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 8155d686..82398216 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -318,6 +318,7 @@ function missingReasonForHeadlineMember( workloadRows: readonly BenchmarkRow[], hardware: string, read: OverviewTierRead, + bucketReads: readonly OverviewTierRead[], ): OverviewMissingReason | null { if (isExactTierRead(read)) return null; const hardwareRows = workloadRows.filter((row) => row.hardware === hardware); @@ -325,7 +326,12 @@ function missingReasonForHeadlineMember( const supportedRows = hardwareRows.filter((row) => OVERVIEW_PRECISIONS.includes(row.precision)); if (supportedRows.length === 0) return 'int4_bf16_only'; if (!supportedRows.some((row) => row.spec_method !== 'none')) return 'standard_decode_only'; - return read.boundary === 'unreachable' ? 'cannot_reach_at_tier' : 'no_exact_at_tier'; + // `cannot reach` is a claim about the whole platform, so it holds only when + // EVERY qualified speculative stack tops out below the tier — one merely + // under-swept stack downgrades the gap to a missing exact read. + return bucketReads.length > 0 && bucketReads.every((r) => r.boundary === 'unreachable') + ? 'cannot_reach_at_tier' + : 'no_exact_at_tier'; } function headlineLeaderTransition( @@ -365,20 +371,23 @@ function buildHeadlinePairs( // lexical dbModel. One bucket per member keeps point releases from blending // within a read. const buildMember = (memberHardware: string): OverviewHeadlinePairMember => { - const best = buildHeadlinePairBuckets(configsByPrecision, new Set([memberHardware]), tier) - .map((memberBucket) => ({ - bucket: memberBucket, - read: nonComparableAsMissing(memberBucket.tierReads.get(memberHardware), tier), - })) - .toSorted( - (a, b) => - Number(isExactTierRead(b.read)) - Number(isExactTierRead(a.read)) || - (b.read.value ?? -1) - (a.read.value ?? -1) || - OVERVIEW_PRECISIONS.indexOf(a.bucket.precision) - - OVERVIEW_PRECISIONS.indexOf(b.bucket.precision) || - b.bucket.newestEvidence.localeCompare(a.bucket.newestEvidence) || - a.bucket.dbModel.localeCompare(b.bucket.dbModel), - )[0]; + const candidates = buildHeadlinePairBuckets( + configsByPrecision, + new Set([memberHardware]), + tier, + ).map((memberBucket) => ({ + bucket: memberBucket, + read: nonComparableAsMissing(memberBucket.tierReads.get(memberHardware), tier), + })); + const best = candidates.toSorted( + (a, b) => + Number(isExactTierRead(b.read)) - Number(isExactTierRead(a.read)) || + (b.read.value ?? -1) - (a.read.value ?? -1) || + OVERVIEW_PRECISIONS.indexOf(a.bucket.precision) - + OVERVIEW_PRECISIONS.indexOf(b.bucket.precision) || + b.bucket.newestEvidence.localeCompare(a.bucket.newestEvidence) || + a.bucket.dbModel.localeCompare(b.bucket.dbModel), + )[0]; const primary = best?.read ?? nullTierRead(tier); const high = nonComparableAsMissing( best === undefined @@ -393,7 +402,12 @@ function buildHeadlinePairs( dbModel: best?.bucket.dbModel ?? null, read: primary, highRead: high, - missingReason: missingReasonForHeadlineMember(workloadRows, memberHardware, primary), + missingReason: missingReasonForHeadlineMember( + workloadRows, + memberHardware, + primary, + candidates.map(({ read }) => read), + ), }; }; From e2b73249e80a2b04c868e02a25762b846d45bfe9 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 23 Jul 2026 10:45:49 -0700 Subject: [PATCH 09/13] feat(overview): mount as first dashboard tab --- packages/app/cypress/component/tab-nav.cy.tsx | 6 ++++ packages/app/cypress/e2e/navigation.cy.ts | 3 ++ packages/app/cypress/e2e/overview.cy.ts | 10 +++++++ .../app/{ => (dashboard)}/overview/page.tsx | 8 +---- .../zh/{ => (dashboard)}/overview/page.tsx | 8 +---- packages/app/src/components/header/header.tsx | 1 + .../src/components/overview/overview-page.tsx | 30 +++++++++---------- packages/app/src/components/tab-nav.tsx | 1 + packages/app/src/lib/tab-meta-zh.ts | 1 + 9 files changed, 39 insertions(+), 29 deletions(-) rename packages/app/src/app/{ => (dashboard)}/overview/page.tsx (87%) rename packages/app/src/app/zh/{ => (dashboard)}/overview/page.tsx (88%) diff --git a/packages/app/cypress/component/tab-nav.cy.tsx b/packages/app/cypress/component/tab-nav.cy.tsx index 2c24d256..df297e61 100644 --- a/packages/app/cypress/component/tab-nav.cy.tsx +++ b/packages/app/cypress/component/tab-nav.cy.tsx @@ -48,6 +48,7 @@ describe('TabNav — unofficialrun URL preservation (issue #319)', () => { it('renders bare hrefs when the URL has no unofficialrun param', () => { mountTabNav({}); + cy.get('[data-testid="tab-trigger-overview"]').should('have.attr', 'href', '/overview'); cy.get('[data-testid="tab-trigger-evaluation"]').should('have.attr', 'href', '/evaluation'); cy.get('[data-testid="tab-trigger-historical"]').should('have.attr', 'href', '/historical'); cy.get('[data-testid="tab-trigger-calculator"]').should('have.attr', 'href', '/calculator'); @@ -55,6 +56,11 @@ describe('TabNav — unofficialrun URL preservation (issue #319)', () => { it('appends unofficialruns to every tab href when the URL has the param', () => { mountTabNav({ search: '?unofficialruns=12345' }); + cy.get('[data-testid="tab-trigger-overview"]').should( + 'have.attr', + 'href', + '/overview?unofficialruns=12345', + ); cy.get('[data-testid="tab-trigger-evaluation"]').should( 'have.attr', 'href', diff --git a/packages/app/cypress/e2e/navigation.cy.ts b/packages/app/cypress/e2e/navigation.cy.ts index c313c81b..47be8122 100644 --- a/packages/app/cypress/e2e/navigation.cy.ts +++ b/packages/app/cypress/e2e/navigation.cy.ts @@ -10,6 +10,9 @@ describe('Chart Section Tabs — E2E', () => { }); it('updates the URL path when switching tabs', () => { + cy.get('[data-testid="tab-trigger-overview"]').click(); + cy.url().should('include', '/overview'); + cy.get('[data-testid="tab-trigger-evaluation"]').click(); cy.url().should('include', '/evaluation'); diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index fc63f3be..2219a98e 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -53,6 +53,12 @@ describe('Overview page', () => { cy.viewport(1280, 900); cy.visit('/overview'); + cy.get('[data-testid="chart-section-tabs"]').should('be.visible'); + cy.get('[data-testid="tab-trigger-overview"]') + .should('have.attr', 'href', '/overview') + .and('have.class', 'border-secondary'); + cy.get('[data-testid="nav-link-dashboard"]').should('have.class', 'text-brand'); + cy.contains('h1', 'AI Inference Overview').should('exist'); cy.contains( 'Every active model across MI355X, B200, B300, GB200 and GB300 at a glance.', @@ -265,6 +271,7 @@ describe('Overview page', () => { cy.viewport(width, 844); cy.visit('/overview'); + cy.get('[data-testid="mobile-chart-select"]').should('be.visible'); cy.get('[data-testid="overview-mobile-list"]').should('be.visible'); cy.get('[data-testid="overview-tier-switcher"]').should('be.visible'); cy.get('[data-testid="overview-desktop-matrix"]').should('not.be.visible'); @@ -287,6 +294,9 @@ describe('Overview page', () => { cy.viewport(1280, 900); cy.visit('/zh/overview'); + cy.get('[data-testid="tab-trigger-overview"]') + .should('have.attr', 'href', '/zh/overview') + .and('contain.text', '总览'); cy.contains('h1', 'AI 推理总览').should('exist'); cy.contains('一眼对比各活跃模型在 MI355X、B200、B300、GB200 与 GB300 上的表现。').should( 'exist', diff --git a/packages/app/src/app/overview/page.tsx b/packages/app/src/app/(dashboard)/overview/page.tsx similarity index 87% rename from packages/app/src/app/overview/page.tsx rename to packages/app/src/app/(dashboard)/overview/page.tsx index 8e1bde73..d1c9f8da 100644 --- a/packages/app/src/app/overview/page.tsx +++ b/packages/app/src/app/(dashboard)/overview/page.tsx @@ -36,11 +36,5 @@ interface Props { export default async function OverviewPage({ searchParams }: Props) { const sp = await searchParams; const data = await getOverviewPageData(resolveOverviewTier(sp.tier)); - return ( -
    -
    - -
    -
    - ); + return ; } diff --git a/packages/app/src/app/zh/overview/page.tsx b/packages/app/src/app/zh/(dashboard)/overview/page.tsx similarity index 88% rename from packages/app/src/app/zh/overview/page.tsx rename to packages/app/src/app/zh/(dashboard)/overview/page.tsx index dd5d6cc9..cd94c065 100644 --- a/packages/app/src/app/zh/overview/page.tsx +++ b/packages/app/src/app/zh/(dashboard)/overview/page.tsx @@ -37,11 +37,5 @@ interface Props { export default async function ZhOverviewPage({ searchParams }: Props) { const sp = await searchParams; const data = await getOverviewPageData(resolveOverviewTier(sp.tier)); - return ( -
    -
    - -
    -
    - ); + return ; } diff --git a/packages/app/src/components/header/header.tsx b/packages/app/src/components/header/header.tsx index e029e445..02877ffd 100644 --- a/packages/app/src/components/header/header.tsx +++ b/packages/app/src/components/header/header.tsx @@ -17,6 +17,7 @@ import { GitHubStars } from './GithubStars'; /** Dashboard tab paths that should highlight the "Dashboard" nav link. */ const DASHBOARD_TABS = [ + '/overview', '/inference', '/evaluation', '/historical', diff --git a/packages/app/src/components/overview/overview-page.tsx b/packages/app/src/components/overview/overview-page.tsx index 5ac8c864..d9171952 100644 --- a/packages/app/src/components/overview/overview-page.tsx +++ b/packages/app/src/components/overview/overview-page.tsx @@ -25,23 +25,23 @@ export function OverviewPageContent({ data, locale }: OverviewPageProps) { : strings.snapshot(formatters.shortDate(data.datasetThroughDate)); return ( -
    -
    -

    {strings.title}

    -

    - {strings.purpose} -

    -

    - {strings.scope(data.tier)} -

    - {snapshot === null ? null : ( -

    {snapshot}

    - )} - -
    +
    + +
    +

    {strings.title}

    +

    {strings.purpose}

    +

    + {strings.scope(data.tier)} +

    + {snapshot === null ? null : ( +

    {snapshot}

    + )} + +
    +
    {/* Official-only summary; uploaded runs remain in the linked dashboard. */} - + = { /** Chinese labels for the dashboard tab bar (TabNav) on /zh pages. */ export const TAB_LABELS_ZH: Record = { + overview: '总览', inference: '推理性能', evaluation: '准确率评估', historical: '历史趋势', From f3d5ae08206849090eacd105e3f2ea2fd706ad22 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 23 Jul 2026 11:02:08 -0700 Subject: [PATCH 10/13] chore(overview): format fixture like other api fixtures --- .prettierignore | 3 - .../cypress/fixtures/api/overview-rows.json | 630 +++++++++++++++++- 2 files changed, 609 insertions(+), 24 deletions(-) delete mode 100644 .prettierignore diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index 3c258190..00000000 --- a/.prettierignore +++ /dev/null @@ -1,3 +0,0 @@ -# Deliberately compact test fixture (one benchmark row per line, <=60 lines); -# oxfmt/prettier would pretty-print it to ~530 lines. See overview-data.test.ts drift guard. -packages/app/cypress/fixtures/api/overview-rows.json diff --git a/packages/app/cypress/fixtures/api/overview-rows.json b/packages/app/cypress/fixtures/api/overview-rows.json index 96a441b0..feb88fe7 100644 --- a/packages/app/cypress/fixtures/api/overview-rows.json +++ b/packages/app/cypress/fixtures/api/overview-rows.json @@ -1,29 +1,617 @@ { "DeepSeek-V4-Pro": [ - {"id": 1, "hardware": "b300", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 1300}, "date": "2026-06-24", "run_url": null}, - {"id": 2, "hardware": "b300", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 70, "output_tput_per_gpu": 1000}, "date": "2026-07-04", "run_url": null}, - {"id": 3, "hardware": "b200", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 40, "output_tput_per_gpu": 1050}, "date": "2026-07-18", "run_url": null}, - {"id": 4, "hardware": "b200", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 110, "output_tput_per_gpu": 300}, "date": "2026-07-18", "run_url": null}, - {"id": 5, "hardware": "mi355x", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 60, "output_tput_per_gpu": 760}, "date": "2026-07-18", "run_url": null}, - {"id": 6, "hardware": "gb300", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 45, "output_tput_per_gpu": 1050}, "date": "2026-07-18", "run_url": null}, - {"id": 7, "hardware": "h100", "framework": "sglang", "model": "dsv4", "precision": "fp4", "spec_method": "none", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 800}, "date": "2026-07-18", "run_url": null}, - {"id": 8, "hardware": "gb200", "framework": "sglang", "model": "dsv4", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null} + { + "id": 1, + "hardware": "b300", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 30, "output_tput_per_gpu": 1300 }, + "date": "2026-06-24", + "run_url": null + }, + { + "id": 2, + "hardware": "b300", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 70, "output_tput_per_gpu": 1000 }, + "date": "2026-07-04", + "run_url": null + }, + { + "id": 3, + "hardware": "b200", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 40, "output_tput_per_gpu": 1050 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 4, + "hardware": "b200", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 110, "output_tput_per_gpu": 300 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 5, + "hardware": "mi355x", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 60, "output_tput_per_gpu": 760 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 6, + "hardware": "gb300", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 45, "output_tput_per_gpu": 1050 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 7, + "hardware": "h100", + "framework": "sglang", + "model": "dsv4", + "precision": "fp4", + "spec_method": "none", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 30, "output_tput_per_gpu": 800 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 8, + "hardware": "gb200", + "framework": "sglang", + "model": "dsv4", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 50, "output_tput_per_gpu": 600 }, + "date": "2026-07-18", + "run_url": null + } ], "Qwen-3.5-397B-A17B": [ - {"id": 9, "hardware": "b300", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 1400}, "date": "2026-07-18", "run_url": null}, - {"id": 10, "hardware": "b300", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 70, "output_tput_per_gpu": 980}, "date": "2026-07-18", "run_url": null}, - {"id": 11, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 40, "output_tput_per_gpu": 800}, "date": "2026-07-18", "run_url": null}, - {"id": 12, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 80, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null}, - {"id": 13, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 900}, "date": "2026-07-18", "run_url": null}, - {"id": 14, "hardware": "mi355x", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 760}, "date": "2026-07-18", "run_url": null}, - {"id": 19, "hardware": "b200", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 120, "output_tput_per_gpu": 300}, "date": "2026-07-18", "run_url": null}, - {"id": 20, "hardware": "mi355x", "framework": "sglang", "model": "qwen3.5", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 24, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 120, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null} + { + "id": 9, + "hardware": "b300", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 30, "output_tput_per_gpu": 1400 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 10, + "hardware": "b300", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 70, "output_tput_per_gpu": 980 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 11, + "hardware": "b200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 40, "output_tput_per_gpu": 800 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 12, + "hardware": "b200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 80, "output_tput_per_gpu": 600 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 13, + "hardware": "b200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 12, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 50, "output_tput_per_gpu": 900 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 14, + "hardware": "mi355x", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 12, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 50, "output_tput_per_gpu": 760 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 19, + "hardware": "b200", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 120, "output_tput_per_gpu": 300 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 20, + "hardware": "mi355x", + "framework": "sglang", + "model": "qwen3.5", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 24, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 120, "output_tput_per_gpu": 600 }, + "date": "2026-07-18", + "run_url": null + } ], "MiniMax-M3": [ - {"id": 15, "hardware": "h200", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 8, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 30, "output_tput_per_gpu": 820}, "date": "2026-07-18", "run_url": null}, - {"id": 16, "hardware": "h200", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 16, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 70, "output_tput_per_gpu": 560}, "date": "2026-07-18", "run_url": null}, - {"id": 17, "hardware": "mi325x", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 650}, "date": "2026-07-18", "run_url": null}, - {"id": 18, "hardware": "h200", "framework": "sglang", "model": "minimaxm3", "precision": "fp4", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 600}, "date": "2026-07-18", "run_url": null}, - {"id": 21, "hardware": "gb300", "framework": "sglang", "model": "minimaxm3", "precision": "fp8", "spec_method": "mtp", "disagg": false, "is_multinode": false, "prefill_tp": 8, "prefill_ep": 1, "prefill_dp_attention": false, "prefill_num_workers": 1, "decode_tp": 8, "decode_ep": 1, "decode_dp_attention": false, "decode_num_workers": 1, "num_prefill_gpu": 8, "num_decode_gpu": 8, "benchmark_type": "single_turn", "isl": 8192, "osl": 1024, "conc": 12, "offload_mode": "off", "image": null, "metrics": {"median_intvty": 50, "output_tput_per_gpu": 700}, "date": "2026-07-18", "run_url": null} + { + "id": 15, + "hardware": "h200", + "framework": "sglang", + "model": "minimaxm3", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 8, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 30, "output_tput_per_gpu": 820 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 16, + "hardware": "h200", + "framework": "sglang", + "model": "minimaxm3", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 16, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 70, "output_tput_per_gpu": 560 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 17, + "hardware": "mi325x", + "framework": "sglang", + "model": "minimaxm3", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 12, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 50, "output_tput_per_gpu": 650 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 18, + "hardware": "h200", + "framework": "sglang", + "model": "minimaxm3", + "precision": "fp4", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 12, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 50, "output_tput_per_gpu": 600 }, + "date": "2026-07-18", + "run_url": null + }, + { + "id": 21, + "hardware": "gb300", + "framework": "sglang", + "model": "minimaxm3", + "precision": "fp8", + "spec_method": "mtp", + "disagg": false, + "is_multinode": false, + "prefill_tp": 8, + "prefill_ep": 1, + "prefill_dp_attention": false, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_dp_attention": false, + "decode_num_workers": 1, + "num_prefill_gpu": 8, + "num_decode_gpu": 8, + "benchmark_type": "single_turn", + "isl": 8192, + "osl": 1024, + "conc": 12, + "offload_mode": "off", + "image": null, + "metrics": { "median_intvty": 50, "output_tput_per_gpu": 700 }, + "date": "2026-07-18", + "run_url": null + } ] } From 69a0a1a3774d7b74421a8f7e3918f0b58a4f56e3 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 23 Jul 2026 11:09:55 -0700 Subject: [PATCH 11/13] chore(overview): trim comments to load-bearing notes --- packages/app/cypress/component/header.cy.tsx | 6 +- packages/app/src/components/header/header.tsx | 7 +- .../overview/overview-detail-link.tsx | 9 +- .../overview/overview-scorecard.tsx | 55 +-------- .../app/src/lib/overview-config-identity.ts | 12 +- packages/app/src/lib/overview-data.server.ts | 7 +- packages/app/src/lib/overview-data.test.ts | 19 +-- packages/app/src/lib/overview-data.ts | 109 ++++-------------- packages/app/src/lib/overview-links.ts | 7 -- 9 files changed, 45 insertions(+), 186 deletions(-) diff --git a/packages/app/cypress/component/header.cy.tsx b/packages/app/cypress/component/header.cy.tsx index 308b31d4..070fcfa8 100644 --- a/packages/app/cypress/component/header.cy.tsx +++ b/packages/app/cypress/component/header.cy.tsx @@ -5,10 +5,8 @@ import { PathnameContext } from 'next/dist/shared/lib/hooks-client-context.share import { Header } from '@/components/header/header'; import { ThemeProvider } from '@/components/ui/theme-provider'; -// Component specs mount outside the Next app shell, so the global stylesheet is -// never injected. The layout assertions below need real Tailwind output, and -// next-style-loader inserts before this anchor node, so both must exist before -// the stylesheet module evaluates. +// Mounted outside the Next app shell; next-style-loader inserts the global +// stylesheet before this anchor, so it must exist before the import below. const cssAnchor = document.createElement('noscript'); cssAnchor.id = '__next_css__DO_NOT_USE__'; document.head.append(cssAnchor); diff --git a/packages/app/src/components/header/header.tsx b/packages/app/src/components/header/header.tsx index 02877ffd..bc438d34 100644 --- a/packages/app/src/components/header/header.tsx +++ b/packages/app/src/components/header/header.tsx @@ -203,11 +203,8 @@ export const Header = ({ starCount }: { starCount?: number | null }) => { - {/* Minecraft-only audio toggles ride alongside GitHub stars below the - same `sm` cutoff: in minecraft mode the pixel font widens the bar, - and these two extra buttons are what push a 320px header past its - bounds. Below `sm` they move into the mobile menu instead (see the - MinecraftToggles instance in the hamburger dropdown below). */} + {/* Below `sm` these move into the mobile menu — they are what push + a 320px header past its bounds in minecraft mode. */} diff --git a/packages/app/src/components/overview/overview-detail-link.tsx b/packages/app/src/components/overview/overview-detail-link.tsx index 2e1e44cb..dbd11283 100644 --- a/packages/app/src/components/overview/overview-detail-link.tsx +++ b/packages/app/src/components/overview/overview-detail-link.tsx @@ -6,12 +6,9 @@ import type { ReactNode } from 'react'; import { track } from '@/lib/analytics'; import { cn } from '@/lib/utils'; -/** - * Model-level dashboard link. A plain anchor, not a ``: the dashboard - * reads its filters from a snapshot `url-state.ts` takes at module evaluation, - * so a client-side navigation would land on an unfiltered dashboard while the - * label promises the model's own view. - */ +/** A plain anchor, not ``: the dashboard reads filters from a snapshot + * `url-state.ts` takes at module evaluation, so client-side navigation would + * land unfiltered. */ export function OverviewDetailLink({ href, model, diff --git a/packages/app/src/components/overview/overview-scorecard.tsx b/packages/app/src/components/overview/overview-scorecard.tsx index a6adb504..20605187 100644 --- a/packages/app/src/components/overview/overview-scorecard.tsx +++ b/packages/app/src/components/overview/overview-scorecard.tsx @@ -1,20 +1,3 @@ -/** - * @file overview-scorecard.tsx - * @description The comparison surface itself: a portfolio matrix of every - * active model (rows) across today's key serving platforms (columns — B200 as - * the reference, then MI355X, B300, GB200 NVL72, GB300 NVL72). The desktop - * table at `xl` and the mobile cards below `xl` render every cell through the - * SAME `BaselineCell`/`CandidateCell` components, so the two surfaces cannot - * drift semantically. - * - * Each cell shows that platform's own best validated speculative read at the - * displayed tier (50 tok/s/user by default; `?tier=` picks another) with its - * precision badge, evidence date and filtered-dashboard link. The signed - * delta against B200 appears only when both displayed reads share a precision - * and a release — mismatches say so instead of comparing FP4 with FP8. A side - * with no exact read shows `∞` with its reason, never a number or percent. - */ - import { OVERVIEW_HIGH_TIER, OVERVIEW_TIERS, @@ -118,11 +101,6 @@ export function overviewFormatters(locale: OverviewLocale): Formatters { }; } -/** - * A result's evidence dates, localized: a single day (`Jul 6` / `7月6日`) when - * both backing frontier points share a date, else an en-dash range - * (`Jun 24–Jul 4`). The caller renders nothing when there is no backing date. - */ function formatEvidenceDate( formatters: Formatters, evidenceDate: { from: string; to: string }, @@ -133,24 +111,18 @@ function formatEvidenceDate( : `${from}–${formatters.shortDate(evidenceDate.to)}`; } -/** The reason copy for a missing side, keyed to the displayed tier. */ function missingReasonCopy(member: OverviewHeadlinePairMember, strings: OverviewStrings): string { const reason = member.missingReason; return reason === null ? '' : strings.missingReasons(member.read.tier)[reason]; } -/** `candidate/B200 − 1` as a rounded percent with a U+2212 or `+` sign. */ function signedPercent(formatters: Formatters, percent: number): string { const rounded = Math.round(percent); const sign = rounded < 0 ? '−' : '+'; return `${sign}${formatters.number.format(Math.abs(rounded))}%`; } -/** - * The @100 line, rendered only when the @100 leader is a different hardware - * than the displayed tier's leader — the leader being the side with the - * greater exact @100 read. Null on the 100 view itself (data layer). - */ +/** Rendered only when the @100 leader flips; the data layer nulls it on the 100 view. */ function highLeadLine( pair: OverviewHeadlinePairComparison, strings: OverviewStrings, @@ -167,7 +139,6 @@ function highLeadLine( const PAIR_VALUE_LINK_CLASS = 'inline-flex min-h-11 items-center rounded-sm underline decoration-dotted underline-offset-4 hover:decoration-solid focus-visible:outline-none focus-visible:ring-[3px] focus-visible:ring-ring/50'; -/** A side with no exact read at the displayed tier: `∞` plus its reason, no link/date/percent. */ function CellMissing({ hardware, reason }: { hardware: string; reason: string }) { return ( ; } - // The full deployment stack behind this read; the framework and precision - // stay visible, the spec method rides the link title and aria only. + // Framework and precision stay visible; the spec method rides the title/aria only. const stack = config === null ? null @@ -250,7 +215,6 @@ function CellValue({ ); } -/** The B200 reference cell: the read alone, no delta against itself. */ function BaselineCell(props: { locale: OverviewLocale; model: OverviewModelSummary; @@ -265,11 +229,6 @@ function BaselineCell(props: { ); } -/** - * One candidate platform's cell: its own read, then the same-precision delta - * against B200, a mismatch note when both sides exist but cannot compare, and - * the @100 leader line only when the leader flips. - */ function CandidateCell({ locale, model, @@ -316,7 +275,6 @@ function CandidateCell({ ); } -/** The model's display name, shared by both surfaces. */ function ModelName({ model }: { model: OverviewModelSummary }) { return

    {model.modelLabel}

    ; } @@ -328,7 +286,6 @@ interface SurfaceProps { strings: OverviewStrings; } -/** Model column, the B200 reference, one column per candidate platform, details. */ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: SurfaceProps) { const headlinePairs = models[0]?.headlinePairs ?? []; const baselineHeader = headlinePairs[0]?.baseline.hardwareLabel ?? null; @@ -417,7 +374,6 @@ export function DesktopOverviewMatrix({ models, locale, formatters, strings }: S ); } -/** Below `xl`: the same cells stacked as one card per model, always fully visible. */ export function MobileOverviewList({ models, locale, formatters, strings }: SurfaceProps) { return (
      @@ -474,11 +430,8 @@ export function MobileOverviewList({ models, locale, formatters, strings }: Surf ); } -/** - * The service-level selector: one plain link per tier, so every view is a - * server-rendered, copyable URL — no client state. The displayed tier is - * inert text marked `aria-current`, never a self-link. - */ +/** Plain links so every view is a copyable server-rendered URL; the displayed + * tier is inert `aria-current` text, never a self-link. */ export function OverviewTierSwitcher({ tier, locale, diff --git a/packages/app/src/lib/overview-config-identity.ts b/packages/app/src/lib/overview-config-identity.ts index 18e4e9f4..c5b5948b 100644 --- a/packages/app/src/lib/overview-config-identity.ts +++ b/packages/app/src/lib/overview-config-identity.ts @@ -1,14 +1,8 @@ import type { BenchmarkRow } from './api'; -/** - * Exact per-deployment identity for the overview grouping. Every dimension that - * makes a serving configuration a distinct deployable topology is part of the - * key — model, hardware, framework, precision, spec method, disagg, multinode, - * per-role parallelism and worker counts, GPU counts, and offload mode. Rows - * that differ only in concurrency, date, image, or run URL share one identity. - * - * Serialized as a JSON tuple so no field delimiter can collide with a value. - */ +/** Every dimension that makes a distinct deployable topology; rows differing + * only in concurrency, date, image, or run URL share one identity. A JSON + * tuple so no delimiter can collide with a value. */ export function overviewConfigIdentityKey(row: BenchmarkRow): string { return JSON.stringify([ row.model, diff --git a/packages/app/src/lib/overview-data.server.ts b/packages/app/src/lib/overview-data.server.ts index 0bfa5fe6..a7fc133c 100644 --- a/packages/app/src/lib/overview-data.server.ts +++ b/packages/app/src/lib/overview-data.server.ts @@ -15,9 +15,8 @@ import { loadFixture } from '@/lib/test-fixtures'; export async function getOverviewPageData( tier: OverviewTier = OVERVIEW_PRIMARY_TIER, ): Promise { - // E2E fixtures mode serves a small synthetic rows-by-display-model fixture - // through the same assembler the live path uses, so a contract drift breaks - // the fixture tests instead of silently stranding the page's data. + // Synthetic rows go through the same assembler as the live path, so a + // contract drift breaks the fixture tests instead of stranding the page. if (FIXTURES_MODE) { return assembleOverviewPageData( loadFixture>('overview-rows'), @@ -25,8 +24,6 @@ export async function getOverviewPageData( ); } - // Fetch rows per db model, concatenated per display model (one display model - // can span several db buckets), then hand the same shape to the assembler. const entries = await Promise.all( [...DEFAULT_MODELS].map(async (model) => { const keys = DISPLAY_MODEL_TO_DB[model] ?? []; diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index 61ba6c7f..da4f03d7 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -482,11 +482,8 @@ describe('tier-parameterized overview', () => { }); }); -// Drift guard for the synthetic overview e2e fixture: if the assembler contract -// changes, the hand-built rows in overview-rows.json must be regenerated, and -// this fails loudly instead of the e2e stranding empty data. Every expectation -// is derived by running this same assembler over the fixture — never eyeballed — -// so it locks the exact matrix cell states overview.cy.ts renders against. +// Drift guard: runs the real assembler over the e2e fixture; expectations are +// engine-derived, never eyeballed. Contract drift fails here, not in overview.cy.ts. describe('assembleOverviewPageData over the overview-rows fixture', () => { it('serves every matrix cell state through the real builder', () => { const page = assembleOverviewPageData( @@ -497,10 +494,8 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { expect(page.datasetThroughDate).toBe('2026-07-18'); expect(page.tier).toBe(50); - // DeepSeek: FP4 exacts on B200/B300 carry the only same-precision delta with - // a cross-day evidence range; GB200's own best is FP8, so its value shows - // but the delta is withheld; MI355X and GB300 miss in opposite clamp - // directions. + // DeepSeek: FP4 delta + cross-day range; GB200's FP8 best withholds its + // delta; MI355X and GB300 miss in opposite clamp directions. const deepseek = page.models.find((m) => m.model === Model.DeepSeek_V4_Pro)!; const dsB300 = headlinePairOf(deepseek, 'b300-vs-b200')!; expect(dsB300.precision).toBe(Precision.FP4); @@ -519,8 +514,7 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { 'cannot_reach_at_tier', ); - // MiniMax: only GB300 has workload rows, so its value stands alone — a - // missing baseline never yields a delta OR a mismatch note. + // MiniMax: a missing baseline yields neither delta nor mismatch note. const minimax = page.models.find((m) => m.model === Model.MiniMax_M3)!; const mmGb300 = headlinePairOf(minimax, 'gb300-vs-b200')!; expect(mmGb300.baseline.missingReason).toBe('no_8k1k_data'); @@ -528,8 +522,7 @@ describe('assembleOverviewPageData over the overview-rows fixture', () => { expect(mmGb300.directDeltaPercent).toBeNull(); expect(mmGb300.deltaUnavailableReason).toBeNull(); - // Qwen: FP8 exacts on both sides yield the −16% delta and a leader flip at - // 100, while B300's FP4 best withholds its delta against the FP8 baseline. + // Qwen: FP8 delta −16% + leader flip at 100; B300's FP4 best withholds its delta. const qwen = page.models.find((m) => m.model === Model.Qwen3_5)!; const qwenMi = headlinePairOf(qwen, 'mi355x-vs-b200')!; expect(qwenMi.precision).toBe(Precision.FP8); diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index 82398216..f74d3fb0 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -10,12 +10,9 @@ import { computeTcoFeed, type TcoTierBoundary } from './tco-feed'; export const OVERVIEW_WORKLOAD = { isl: 8192, osl: 1024 } as const; export const OVERVIEW_TIERS = [30, 50, 75, 100] as const; export type OverviewTier = (typeof OVERVIEW_TIERS)[number]; -/** Default service point; ?tier= re-renders the matrix at any OVERVIEW_TIERS value. */ export const OVERVIEW_PRIMARY_TIER = 50; -/** High-interactivity capability read; also drives the high-tier leader transition. */ export const OVERVIEW_HIGH_TIER = 100; -/** The `?tier=` query value as a displayable tier; anything else falls back to 50. */ export function resolveOverviewTier(raw: string | string[] | undefined): OverviewTier { const candidate = Number(Array.isArray(raw) ? raw[0] : raw); return OVERVIEW_TIERS.find((tier) => tier === candidate) ?? OVERVIEW_PRIMARY_TIER; @@ -25,22 +22,13 @@ export interface OverviewTierValue { tier: number; value: number | null; boundary: TcoTierBoundary; - /** - * Run dates of the frontier point(s) backing this value — the two bracketing - * points when interpolated (from = earlier, to = later; equal on the same - * day), the single point twice when clamped, null when `value` is null. Comes - * only from this config's own frontier at this tier, never a sibling or cohort. - */ + /** Bracketing frontier points when interpolated, the single point twice when + * clamped; always from this config's own frontier, never a sibling's. */ evidenceDate: { from: string; to: string } | null; } -/** - * One real deployable serving configuration, identified by its exact - * deployment topology (model × hardware × framework × precision × spec_method × - * disagg × multinode × per-role parallelism × GPU counts × offload). Tier - * values come from this configuration's own Pareto frontier only — never - * blended across configurations. - */ +/** One deployable serving configuration (exact topology identity); tier values + * come from its own Pareto frontier only, never blended across configs. */ export interface OverviewConfigResult { key: string; dbModel: string; @@ -57,28 +45,17 @@ export interface OverviewConfigResult { latestDate: string; } -/** One hardware's frontier read at a single tier, with its backing config. */ export interface OverviewTierRead { tier: number; value: number | null; boundary: TcoTierBoundary | null; - /** Copied from the backing config's tier value; null when there is no read. */ evidenceDate: { from: string; to: string } | null; config: OverviewConfigResult | null; } -/** - * Why a pair member shows `∞` instead of a value, from most to least - * fundamental gap: - * - * - `no_8k1k_data` — no 8K/1K single-turn row for this hardware at all; - * - `int4_bf16_only` — measured only outside FP4/FP8; - * - `standard_decode_only` — FP4/FP8 measured, but standard decode only; - * - `cannot_reach_at_tier` — speculative frontier tops out below the - * displayed tier; - * - `no_exact_at_tier` — no exact read at the displayed tier for any other - * reason (an under-swept frontier that starts above the tier). - */ +/** Why a member shows `∞`. The subtle pair: `cannot_reach_at_tier` = every + * speculative stack tops out below the tier; `no_exact_at_tier` = merely + * under-swept. */ export type OverviewMissingReason = | 'standard_decode_only' | 'int4_bf16_only' @@ -98,18 +75,10 @@ export type OverviewHeadlinePairId = (typeof OVERVIEW_HEADLINE_PAIR_DEFINITIONS) export interface OverviewHeadlinePairMember { hardware: string; hardwareLabel: string; - /** - * The precision of this platform's own best qualified speculative read at - * the displayed tier, selected independently per platform (exact first, then - * value, tie → FP4). Null when the platform has no FP4/FP8 speculative - * bucket at all. - */ precision: string | null; - /** The dbModel (point release) backing this platform's selected read. */ dbModel: string | null; - /** Exact read when available; non-exact reads retain their boundary but never a numeric value. */ read: OverviewTierRead; - /** Best @100 read from this platform's own selected dbModel and precision. */ + /** @100 read from the same selected bucket as `read`. */ highRead: OverviewTierRead; missingReason: OverviewMissingReason | null; } @@ -117,19 +86,13 @@ export interface OverviewHeadlinePairMember { export interface OverviewHeadlinePairComparison { id: OverviewHeadlinePairId; label: string; - /** The precision shared by both displayed reads when the delta is comparable; null otherwise. */ precision: string | null; - /** The dbModel shared by both displayed reads when the delta is comparable; null otherwise. */ dbModel: string | null; candidate: OverviewHeadlinePairMember; baseline: OverviewHeadlinePairMember; - /** - * Candidate relative to B200. Non-null only when both displayed-tier reads - * are exact, share one precision AND one dbModel, and B200 > 0 — a delta - * never mixes FP4 with FP8 or two point releases. - */ + /** Non-null only for exact reads sharing precision AND dbModel — never FP4 + * vs FP8 or cross-release. */ directDeltaPercent: number | null; - /** Why two displayed exact reads still carry no delta; null when a delta exists or a side is missing. */ deltaUnavailableReason: 'precision_mismatch' | 'version_mismatch' | null; highLeaderTransition: 'same_hardware' | 'changed_hardware' | null; } @@ -137,21 +100,18 @@ export interface OverviewHeadlinePairComparison { export interface OverviewModelSummary { model: Model; modelLabel: string; - /** Fixed candidate-vs-B200 serving-stack comparisons, one per matrix column. */ headlinePairs: OverviewHeadlinePairComparison[]; } export interface OverviewPageData { models: OverviewModelSummary[]; datasetThroughDate: string | null; - /** The service point every headline pair was read at. */ tier: OverviewTier; } -/** Precisions the overview may rank, in preference order. */ +/** In preference order — FP4 wins ties. */ const OVERVIEW_PRECISIONS: readonly string[] = [Precision.FP4, Precision.FP8]; -/** Rows measuring the fixed overview workload, before any other filter. */ function overviewWorkloadRows(rows: readonly BenchmarkRow[]): BenchmarkRow[] { return rows.filter( (row) => @@ -161,11 +121,8 @@ function overviewWorkloadRows(rows: readonly BenchmarkRow[]): BenchmarkRow[] { ); } -/** - * Newest raw workload row across the page. Deliberately not derived from the - * retained winners: a precision or engine the page never ranks still dates the - * dataset it was measured in. - */ +/** Deliberately from raw rows, not retained winners: an unranked precision or + * engine still dates the dataset it was measured in. */ export function overviewDatasetThroughDate(rows: readonly BenchmarkRow[]): string | null { return overviewWorkloadRows(rows).reduce( (latest, row) => (latest === null || row.date > latest ? row.date : latest), @@ -173,11 +130,7 @@ export function overviewDatasetThroughDate(rows: readonly BenchmarkRow[]): strin ); } -/** - * Speculative configs for one precision, grouped by exact deployment identity. - * Standard-decode rows never enter the pool, so a precision is ranked on its - * speculative frontier alone. - */ +/** Speculative-only configs for one precision, grouped by exact deployment identity. */ function buildPrecisionConfigs( model: Model, workloadRows: readonly BenchmarkRow[], @@ -211,16 +164,12 @@ function readConfigAtTier(config: OverviewConfigResult, tier: number): OverviewT }; } -/** A tier read known to be backed by a configuration. */ interface ConfigTierRead extends OverviewTierRead { config: OverviewConfigResult; } -/** - * A tier read counts only when it lands inside the configuration's own measured - * frontier. A clamped or unreachable read is a coverage gap, so it can never - * lead a tier or anchor a percentage gap. - */ +/** In-range reads only: a clamped/unreachable read is a coverage gap and never + * leads a tier or anchors a delta. */ const isExactTierRead = (read: T): read is T & { value: number } => read.value !== null && read.boundary === 'interpolated'; @@ -232,11 +181,8 @@ function compareTierReads(a: ConfigTierRead, b: ConfigTierRead): number { ); } -/** - * One read per hardware at `tier`: its best exact in-range read, or — only when - * the hardware has no exact read at all — its best out-of-range read, kept so - * the hardware still surfaces as a coverage gap rather than disappearing. - */ +/** Best exact read per hardware; a hardware with no exact read keeps its best + * out-of-range read so it surfaces as a gap instead of disappearing. */ function readsByHardwareAtTier( configs: readonly OverviewConfigResult[], tier: number, @@ -360,16 +306,14 @@ function buildHeadlinePairs( workloadRows: readonly BenchmarkRow[], tier: OverviewTier, ): OverviewHeadlinePairComparison[] { - // FP4 and FP8 configs are built independently and never blend within a read. const configsByPrecision = new Map( OVERVIEW_PRECISIONS.map( (precision) => [precision, buildPrecisionConfigs(model, workloadRows, precision)] as const, ), ); - // Each platform selects its own best qualified bucket (dbModel × precision): - // exact at the displayed tier first, then value, tie → FP4, newest evidence, - // lexical dbModel. One bucket per member keeps point releases from blending - // within a read. + // Per-platform best bucket (dbModel × precision): exact first, then value, + // tie → FP4, newest evidence, lexical dbModel — one bucket per member so + // point releases never blend within a read. const buildMember = (memberHardware: string): OverviewHeadlinePairMember => { const candidates = buildHeadlinePairBuckets( configsByPrecision, @@ -440,8 +384,6 @@ function buildHeadlinePairs( : 'version_mismatch' : 'precision_mismatch' : null, - // On the 100 view the displayed reads ARE the high reads, so a - // transition line would restate the cell values. highLeaderTransition: comparable && tier !== OVERVIEW_HIGH_TIER ? headlineLeaderTransition( @@ -509,13 +451,8 @@ export function buildOverviewModelSummary( }; } -/** - * Assemble the whole page from rows already grouped by DISPLAY model. Iterating - * DEFAULT_MODELS fixes the output order and renders every active model — a model - * with no rows still gets its four pairs, every member carrying a missing - * reason. The live server path and the e2e fixture path feed this same - * function; only the row source differs. - */ +/** DEFAULT_MODELS fixes the row order; a rowless model still renders four + * pairs with missing reasons. Live and fixture paths both feed this. */ export function assembleOverviewPageData( rowsByModel: Record, tier: OverviewTier = OVERVIEW_PRIMARY_TIER, diff --git a/packages/app/src/lib/overview-links.ts b/packages/app/src/lib/overview-links.ts index 132def11..ea8e34d1 100644 --- a/packages/app/src/lib/overview-links.ts +++ b/packages/app/src/lib/overview-links.ts @@ -1,10 +1,3 @@ -/** - * @file overview-links.ts - * @description Evidence links for the overview scorecard: a pre-filtered - * /inference view and the CI run a configuration came from. Pure and - * React-free so the server-rendered overview page can build them. - */ - import { runIdFromRunUrl } from './known-issues'; import { OVERVIEW_PRIMARY_TIER, From b507220ec84cac2acbf1923957903a3c7a1baa16 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 23 Jul 2026 11:22:20 -0700 Subject: [PATCH 12/13] fix(overview): leader line follows the 100 view's own re-selection --- packages/app/src/lib/overview-data.test.ts | 15 ++++++++ packages/app/src/lib/overview-data.ts | 44 +++++++++++++--------- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/packages/app/src/lib/overview-data.test.ts b/packages/app/src/lib/overview-data.test.ts index da4f03d7..07f2832f 100644 --- a/packages/app/src/lib/overview-data.test.ts +++ b/packages/app/src/lib/overview-data.test.ts @@ -294,6 +294,21 @@ describe('overview headline pairs', () => { expect(pair?.highLeaderTransition).toBe('changed_hardware'); }); + it('suppresses the leader line when another precision wins the 100 view', () => { + // Both sides are FP8-comparable @50, but the candidate's FP4 stack wins + // @100 — the 100 view re-selects FP4 vs FP8 and refuses to compare. + const summary = buildOverviewModelSummary(Model.Qwen3_5, [ + ...frontier([1100, 900, 700, 500], { hardware: 'mi355x', precision: Precision.FP8 }), + ...frontier([1000, 800, 750, 700], { hardware: 'mi355x', precision: Precision.FP4 }), + ...frontier([1050, 850, 720, 600], { hardware: 'b200', precision: Precision.FP8 }), + ]); + + const pair = headlinePairOf(summary, 'mi355x-vs-b200'); + expect(pair?.directDeltaPercent).toBeCloseTo((900 / 850 - 1) * 100); + expect(pair?.candidate.highRead.config?.precision).toBe(Precision.FP4); + expect(pair?.highLeaderTransition).toBeNull(); + }); + it('claims cannot-reach only when every speculative bucket is unreachable', () => { const unreachable: [number, number][] = [ [20, 500], diff --git a/packages/app/src/lib/overview-data.ts b/packages/app/src/lib/overview-data.ts index f74d3fb0..0c2c47ca 100644 --- a/packages/app/src/lib/overview-data.ts +++ b/packages/app/src/lib/overview-data.ts @@ -78,7 +78,7 @@ export interface OverviewHeadlinePairMember { precision: string | null; dbModel: string | null; read: OverviewTierRead; - /** @100 read from the same selected bucket as `read`. */ + /** The 100 view's own re-selected read, so the leader line can never disagree with ?tier=100. */ highRead: OverviewTierRead; missingReason: OverviewMissingReason | null; } @@ -286,11 +286,17 @@ function headlineLeaderTransition( candidateHigh: OverviewTierRead, baselineHigh: OverviewTierRead, ): OverviewHeadlinePairComparison['highLeaderTransition'] { + // A cross-precision or cross-release @100 pair carries no leader claim, + // mirroring the delta rule. if ( !isExactTierRead(candidatePrimary) || !isExactTierRead(baselinePrimary) || !isExactTierRead(candidateHigh) || !isExactTierRead(baselineHigh) || + candidateHigh.config === null || + baselineHigh.config === null || + candidateHigh.config.precision !== baselineHigh.config.precision || + candidateHigh.config.dbModel !== baselineHigh.config.dbModel || candidatePrimary.value === baselinePrimary.value || candidateHigh.value === baselineHigh.value ) { @@ -314,14 +320,14 @@ function buildHeadlinePairs( // Per-platform best bucket (dbModel × precision): exact first, then value, // tie → FP4, newest evidence, lexical dbModel — one bucket per member so // point releases never blend within a read. - const buildMember = (memberHardware: string): OverviewHeadlinePairMember => { + const selectRead = (memberHardware: string, atTier: OverviewTier) => { const candidates = buildHeadlinePairBuckets( configsByPrecision, new Set([memberHardware]), - tier, + atTier, ).map((memberBucket) => ({ bucket: memberBucket, - read: nonComparableAsMissing(memberBucket.tierReads.get(memberHardware), tier), + read: nonComparableAsMissing(memberBucket.tierReads.get(memberHardware), atTier), })); const best = candidates.toSorted( (a, b) => @@ -332,25 +338,29 @@ function buildHeadlinePairs( b.bucket.newestEvidence.localeCompare(a.bucket.newestEvidence) || a.bucket.dbModel.localeCompare(b.bucket.dbModel), )[0]; - const primary = best?.read ?? nullTierRead(tier); - const high = nonComparableAsMissing( - best === undefined - ? undefined - : readsByHardwareAtTier(best.bucket.configs, OVERVIEW_HIGH_TIER).get(memberHardware), - OVERVIEW_HIGH_TIER, - ); + return { + reads: candidates.map(({ read }) => read), + bucket: best?.bucket ?? null, + read: best?.read ?? nullTierRead(atTier), + }; + }; + + const buildMember = (memberHardware: string): OverviewHeadlinePairMember => { + const primary = selectRead(memberHardware, tier); + const high = + tier === OVERVIEW_HIGH_TIER ? primary : selectRead(memberHardware, OVERVIEW_HIGH_TIER); return { hardware: memberHardware, hardwareLabel: getHardwareConfig(memberHardware, model).label, - precision: best?.bucket.precision ?? null, - dbModel: best?.bucket.dbModel ?? null, - read: primary, - highRead: high, + precision: primary.bucket?.precision ?? null, + dbModel: primary.bucket?.dbModel ?? null, + read: primary.read, + highRead: high.read, missingReason: missingReasonForHeadlineMember( workloadRows, memberHardware, - primary, - candidates.map(({ read }) => read), + primary.read, + primary.reads, ), }; }; From 5aade7294ebe86422c47530fcd1ac0c449a6323c Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 23 Jul 2026 11:22:33 -0700 Subject: [PATCH 13/13] fix(header): carry query state into the language toggle href --- packages/app/cypress/e2e/overview.cy.ts | 4 +++- packages/app/src/components/header/header.tsx | 12 ++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/app/cypress/e2e/overview.cy.ts b/packages/app/cypress/e2e/overview.cy.ts index 2219a98e..955f1249 100644 --- a/packages/app/cypress/e2e/overview.cy.ts +++ b/packages/app/cypress/e2e/overview.cy.ts @@ -260,7 +260,9 @@ describe('Overview page', () => { cy.visit('/overview?tier=100'); cy.contains('Output tok/s/GPU @100 tok/s/user').should('exist'); - cy.get('[data-testid="language-toggle"]').click(); + cy.get('[data-testid="language-toggle"]') + .should('have.attr', 'href', '/zh/overview?tier=100') + .click(); cy.location('pathname').should('eq', '/zh/overview'); cy.location('search').should('eq', '?tier=100'); cy.contains('每 GPU 输出 tok/s @100 tok/s/用户').should('exist'); diff --git a/packages/app/src/components/header/header.tsx b/packages/app/src/components/header/header.tsx index bc438d34..021d5795 100644 --- a/packages/app/src/components/header/header.tsx +++ b/packages/app/src/components/header/header.tsx @@ -85,15 +85,23 @@ function LanguageToggle({ }) { const isZh = isZhPathname(pathname); const target = switchLocalePath(pathname); + // The href carries the query too, so modified clicks and copied links keep + // shareable state (e.g. the overview's ?tier=) — same sync as TabNav's. + const [search, setSearch] = useState(''); + useEffect(() => { + const sync = () => setSearch(window.location.search); + sync(); + window.addEventListener('popstate', sync); + return () => window.removeEventListener('popstate', sync); + }, [pathname]); return ( { track('header_language_toggled', { to: isZh ? 'en' : 'zh' }); - // The sibling keeps shareable query state (e.g. the overview's ?tier=). navigateInApp(event, router, target + window.location.search); }} >