diff --git a/packages/app/src/components/ai-chart/prompt-templates.ts b/packages/app/src/components/ai-chart/prompt-templates.ts index 59beedfe..b72b1958 100644 --- a/packages/app/src/components/ai-chart/prompt-templates.ts +++ b/packages/app/src/components/ai-chart/prompt-templates.ts @@ -51,8 +51,8 @@ Open-source ML inference benchmark dashboard. Automated CI runs test real servin **Sequences**: 1k/1k, 1k/8k, 8k/1k (input/output token lengths) **Benchmark y-axis metrics**: ${Y_METRIC_LIST} - Throughput: y_tpPerGpu (total tok/s/gpu, DEFAULT), y_outputTputPerGpu, y_inputTputPerGpu - Efficiency: y_tpPerMw (tok/s/MW) + Throughput: y_tpPerGpu (total tok/s/gpu, DEFAULT), y_outputTputPerGpu, y_inputTputPerGpu, y_outputTputPerTotalGpu (output tok/s over total GPUs incl. prefill) + Efficiency: y_tpPerMw (tok/s/MW), y_outputTputPerTotalMw (output tok/s/MW over total GPUs) Cost: y_costh (hyperscaler $/Mtok), y_costn (neocloud), y_costr (3yr rental) Energy: y_jTotal (J/tok), y_jOutput, y_jInput **Eval metric**: eval_score diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index cde86370..2990ae09 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -37,6 +37,14 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { const specs = getGpuSpecs(hwKey); const power = specs.power; + // Output throughput normalized over the total GPU count — mirrors the + // outputTputPerTotalGpu derivation in createChartDataPoint (chart-utils.ts). + const totalGpuCount = entry.num_prefill_gpu + entry.num_decode_gpu; + const outputTputTotal = + entry.disagg && totalGpuCount > 0 + ? (outputTput * entry.num_decode_gpu) / totalGpuCount + : outputTput; + const tokPerHr = (tput * 3600) / 1_000_000; const outTokPerHr = (outputTput * 3600) / 1_000_000; const inTokPerHr = (inputTput * 3600) / 1_000_000; @@ -55,8 +63,10 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { date: row.date, tpPerGpu: wrapMetric(tput), outputTputPerGpu: wrapMetric(outputTput), + outputTputPerTotalGpu: wrapMetric(outputTputTotal), inputTputPerGpu: wrapMetric(inputTput), tpPerMw: wrapMetric(power > 0 ? (tput * 1000) / power : 0), + outputTputPerTotalMw: wrapMetric(power > 0 ? (outputTputTotal * 1000) / power : 0), // Cost per million tokens (total / output / input) costh: wrapMetric(tokPerHr ? specs.costh / tokPerHr : 0), costn: wrapMetric(tokPerHr ? specs.costn / tokPerHr : 0), diff --git a/packages/app/src/components/inference/inference-chart-config.json b/packages/app/src/components/inference/inference-chart-config.json index e161a43a..c7a78bce 100644 --- a/packages/app/src/components/inference/inference-chart-config.json +++ b/packages/app/src/components/inference/inference-chart-config.json @@ -26,6 +26,12 @@ "y_outputTputPerGpu_title": "Output Token Throughput per GPU", "y_outputTputPerGpu_titleZh": "每 GPU 输出 token 吞吐量", "y_outputTputPerGpu_roofline": "upper_left", + "y_outputTputPerTotalGpu": "outputTputPerTotalGpu.y", + "y_outputTputPerTotalGpu_label": "Output Token Throughput per Total GPUs (tok/s/gpu)", + "y_outputTputPerTotalGpu_labelZh": "每 GPU 输出 token 吞吐量（按总 GPU 数）（tok/s/gpu）", + "y_outputTputPerTotalGpu_title": "Output Token Throughput per Total GPUs", + "y_outputTputPerTotalGpu_titleZh": "每 GPU 输出 token 吞吐量（按总 GPU 数）", + "y_outputTputPerTotalGpu_roofline": "upper_left", "y_tpPerMw": "tpPerMw.y", "y_tpPerMw_label": "Token Throughput per All in Utility MW (tok/s/MW)", "y_tpPerMw_labelZh": "每全电源配置兆瓦 token 吞吐量（tok/s/MW）", @@ -44,6 +50,12 @@ "y_outputTputPerMw_title": "Output Token Throughput per All in Utility MW", "y_outputTputPerMw_titleZh": "每全电源配置兆瓦输出 token 吞吐量", "y_outputTputPerMw_roofline": "upper_left", + "y_outputTputPerTotalMw": "outputTputPerTotalMw.y", + "y_outputTputPerTotalMw_label": "Output Token Throughput per All in Utility MW, Total GPUs (tok/s/MW)", + "y_outputTputPerTotalMw_labelZh": "每全电源配置兆瓦输出 token 吞吐量（按总 GPU 数）（tok/s/MW）", + "y_outputTputPerTotalMw_title": "Output Token Throughput per All in Utility MW (Total GPUs)", + "y_outputTputPerTotalMw_titleZh": "每全电源配置兆瓦输出 token 吞吐量（按总 GPU 数）", + "y_outputTputPerTotalMw_roofline": "upper_left", "y_costh": "costh.y", "y_costh_label": "Cost per Million Total Tokens ($)", "y_costh_labelZh": "每百万总 token 成本（$）", @@ -190,6 +202,12 @@ "y_outputTputPerGpu_title": "Output Token Throughput per GPU", "y_outputTputPerGpu_titleZh": "每 GPU 输出 token 吞吐量", "y_outputTputPerGpu_roofline": "upper_right", + "y_outputTputPerTotalGpu": "outputTputPerTotalGpu.y", + "y_outputTputPerTotalGpu_label": "Output Token Throughput per Total GPUs (tok/s/gpu)", + "y_outputTputPerTotalGpu_labelZh": "每 GPU 输出 token 吞吐量（按总 GPU 数）（tok/s/gpu）", + "y_outputTputPerTotalGpu_title": "Output Token Throughput per Total GPUs", + "y_outputTputPerTotalGpu_titleZh": "每 GPU 输出 token 吞吐量（按总 GPU 数）", + "y_outputTputPerTotalGpu_roofline": "upper_right", "y_tpPerMw": "tpPerMw.y", "y_tpPerMw_label": "Token Throughput per All in Utility MW (tok/s/MW)", "y_tpPerMw_labelZh": "每全电源配置兆瓦 token 吞吐量（tok/s/MW）", @@ -208,6 +226,12 @@ "y_outputTputPerMw_title": "Output Token Throughput per All in Utility MW", "y_outputTputPerMw_titleZh": "每全电源配置兆瓦输出 token 吞吐量", "y_outputTputPerMw_roofline": "upper_right", + "y_outputTputPerTotalMw": "outputTputPerTotalMw.y", + "y_outputTputPerTotalMw_label": "Output Token Throughput per All in Utility MW, Total GPUs (tok/s/MW)", + "y_outputTputPerTotalMw_labelZh": "每全电源配置兆瓦输出 token 吞吐量（按总 GPU 数）（tok/s/MW）", + "y_outputTputPerTotalMw_title": "Output Token Throughput per All in Utility MW (Total GPUs)", + "y_outputTputPerTotalMw_titleZh": "每全电源配置兆瓦输出 token 吞吐量（按总 GPU 数）", + "y_outputTputPerTotalMw_roofline": "upper_right", "y_costh": "costh.y", "y_costh_label": "Cost per Million Total Tokens ($)", "y_costh_labelZh": "每百万总 token 成本（$）", diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8b45e0f7..3fe75fcc 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -274,10 +274,12 @@ export interface InferenceData extends Partial { expect(result[0].y).toBe(5); }); + it('remaps y to outputTputPerTotalGpu for unofficial-run overlay points', () => { + // Overlay points flow through the same transform pipeline, so the total-GPU + // metrics must resolve for ?unofficialrun= overlays like any other metric. + const data = [pt({ outputTputPerTotalGpu: { y: 300, roof: false }, median_intvty: 10 } as any)]; + const result = processOverlayChartData(data, 'interactivity', 'y_outputTputPerTotalGpu', null); + expect(result).toHaveLength(1); + expect(result[0].y).toBe(300); + }); + + it('remaps y to outputTputPerTotalMw and drops overlay points missing it', () => { + const withMetric = pt({ + outputTputPerTotalMw: { y: 425, roof: false }, + median_intvty: 10, + } as any); + const withoutMetric = pt({ median_intvty: 10 } as any); + const result = processOverlayChartData( + [withMetric, withoutMetric], + 'interactivity', + 'y_outputTputPerTotalMw', + null, + ); + expect(result).toHaveLength(1); + expect(result[0].y).toBe(425); + }); + it('remaps x to config override for input metrics on interactivity chart', () => { // inputTputPerGpu has x override to p90_ttft on interactivity chart const data = [ diff --git a/packages/app/src/components/ui/chart-display-helpers.test.tsx b/packages/app/src/components/ui/chart-display-helpers.test.tsx index f78b570a..47db64dd 100644 --- a/packages/app/src/components/ui/chart-display-helpers.test.tsx +++ b/packages/app/src/components/ui/chart-display-helpers.test.tsx @@ -104,4 +104,41 @@ describe('MetricAssumptionNotes', () => { expect(getVisibleCaveatText()).toContain('calculate Joules per decode GPU or per prefill GPU'); expect(getVisibleCaveatText()).toContain('direct Joules per token comparison'); }); + + it('shows power source badges without the per-pool disagg caveat for y_outputTputPerTotalMw', () => { + renderUi(); + + // Power badges apply — the metric divides by the spec-sheet all-in power… + expect(getVisibleText()).toContain('All in Power/GPU:'); + expect(getVisibleText()).toContain('SemiAnalysis Datacenter Industry Model'); + // …but the per-decode/prefill-pool caveat does NOT: this metric already + // normalizes over the total GPU count. + expect(getVisibleCaveatText()).not.toContain( + 'calculate power per decode GPU or per prefill GPU', + ); + expect(getVisibleCaveatText()).toContain( + 'normalizes output throughput over the total GPU count', + ); + }); + + it('shows the total-GPU normalization note for y_outputTputPerTotalGpu only', () => { + renderUi(); + + expect(getVisibleCaveatText()).toContain( + 'normalizes output throughput over the total GPU count', + ); + // The per-decode-GPU caveat belongs to y_outputTputPerGpu, not this metric + expect(getVisibleCaveatText()).not.toContain( + 'calculate output throughput per decode GPU or per prefill GPU', + ); + + renderUi(); + + expect(getVisibleCaveatText()).toContain( + 'calculate output throughput per decode GPU or per prefill GPU', + ); + expect(getVisibleCaveatText()).not.toContain( + 'normalizes output throughput over the total GPU count', + ); + }); }); diff --git a/packages/app/src/components/ui/chart-display-helpers.tsx b/packages/app/src/components/ui/chart-display-helpers.tsx index 5c7d97c4..293c9c3a 100644 --- a/packages/app/src/components/ui/chart-display-helpers.tsx +++ b/packages/app/src/components/ui/chart-display-helpers.tsx @@ -12,7 +12,19 @@ import type { Locale } from '@/lib/i18n'; // Keep these metric-key groups in sync with chart-utils/chart configs when new source-backed // metrics are added; this helper owns which caption notes and caveats appear for each family. -const POWER_SOURCE_METRICS = new Set(['y_tpPerMw', 'y_inputTputPerMw', 'y_outputTputPerMw']); +const POWER_SOURCE_METRICS = new Set([ + 'y_tpPerMw', + 'y_inputTputPerMw', + 'y_outputTputPerMw', + 'y_outputTputPerTotalMw', +]); +// Per-MW metrics that normalize disagg configs per decode/prefill GPU pool only. +// y_outputTputPerTotalMw is excluded — it already folds in the total GPU count, +// so the per-pool disagg caveat doesn't apply (it gets TotalGpuNote instead). +const PER_POOL_POWER_METRICS = new Set(['y_tpPerMw', 'y_inputTputPerMw', 'y_outputTputPerMw']); +// Metrics normalized over the total GPU count (prefill + decode) — disagg configs +// ARE apples-to-apples with aggregated configs here, which TotalGpuNote calls out. +const TOTAL_GPU_NORMALIZED_METRICS = new Set(['y_outputTputPerTotalGpu', 'y_outputTputPerTotalMw']); const TOTAL_COST_METRICS = new Set(['y_costh', 'y_costn', 'y_costr']); const OUTPUT_COST_METRICS = new Set(['y_costhOutput', 'y_costnOutput', 'y_costrOutput']); const INPUT_COST_METRICS = new Set(['y_costhi', 'y_costni', 'y_costri']); @@ -113,6 +125,34 @@ function DisaggCaveat({ ); } +function TotalGpuNote({ visible, locale = 'en' }: { visible: boolean; locale?: Locale }) { + const content = + locale === 'zh' ? ( + <> + 注意：该指标按总 GPU 数（预填充 + 解码 GPU）对输出吞吐量进行归一化， + 因此分离式推理配置可与聚合配置直接对比。 + + ) : ( + <> + Note: This metric normalizes output throughput over the total GPU count + (prefill + decode GPUs), so disaggregated inference configurations are directly comparable + with aggregated configs. + + ); + + return ( +
+

+ {content} +

+
+ ); +} + function getCostValues(selectedYAxisMetric: string) { return Object.fromEntries( Object.entries(HW_REGISTRY).map(([base, specs]) => [ @@ -203,11 +243,15 @@ export function MetricAssumptionNotes({ /> {includePowerThroughputCaveat && ( )} + {showJouleSource && ( <> diff --git a/packages/app/src/hooks/api/use-ai-chart.ts b/packages/app/src/hooks/api/use-ai-chart.ts index 669935cf..d8a7344d 100644 --- a/packages/app/src/hooks/api/use-ai-chart.ts +++ b/packages/app/src/hooks/api/use-ai-chart.ts @@ -244,8 +244,10 @@ function buildReliabilityBarData( const METRIC_LABELS: Record = { y_tpPerGpu: 'Throughput/GPU', y_outputTputPerGpu: 'Output Tput/GPU', + y_outputTputPerTotalGpu: 'Output Tput/Total GPUs', y_inputTputPerGpu: 'Input Tput/GPU', y_tpPerMw: 'Tput/MW', + y_outputTputPerTotalMw: 'Output Tput/MW (Total GPUs)', y_costh: 'Cost (Hyper)', y_costn: 'Cost (Neo)', y_costr: 'Cost (Rental)', diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index 22a16af3..0011bbdd 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -762,6 +762,51 @@ describe('transformBenchmarkRows — data point values', () => { expect(point.inputTputPerGpu!.y).toBe(80); }); + it('sets outputTputPerTotalGpu equal to outputTputPerGpu for non-disagg rows', () => { + const rows = [ + makeRow({ + disagg: false, + metrics: { + tput_per_gpu: 500, + output_tput_per_gpu: 420, + input_tput_per_gpu: 80, + median_intvty: 18, + median_e2el: 2.5, + }, + }), + ]; + const { chartData } = transformBenchmarkRows(rows); + const point = chartData[0][0]; + expect(point.outputTputPerTotalGpu).toBeDefined(); + expect(point.outputTputPerTotalGpu!.y).toBe(420); + expect(point.outputTputPerTotalMw).toBeDefined(); + expect(point.outputTputPerTotalMw!.y).toBeGreaterThan(0); + }); + + it('normalizes outputTputPerTotalGpu over prefill+decode GPUs for disagg rows', () => { + const rows = [ + makeRow({ + disagg: true, + num_prefill_gpu: 4, + num_decode_gpu: 2, + metrics: { + tput_per_gpu: 500, + output_tput_per_gpu: 900, + input_tput_per_gpu: 80, + median_intvty: 18, + median_e2el: 2.5, + }, + }), + ]; + const { chartData } = transformBenchmarkRows(rows); + const point = chartData[0][0]; + // (900 tok/s/decode-gpu * 2 decode GPUs) / (4 + 2 total GPUs) + expect(point.outputTputPerTotalGpu).toBeDefined(); + expect(point.outputTputPerTotalGpu!.y).toBe(300); + // Per-decode-GPU metric stays untouched + expect(point.outputTputPerGpu!.y).toBe(900); + }); + it('groups data points by hwKey within each chart', () => { const rows = [ makeRow({ hardware: 'h200', framework: 'trt', conc: 16 }), diff --git a/packages/app/src/lib/chart-utils.test.ts b/packages/app/src/lib/chart-utils.test.ts index cfde6ded..d39966cc 100644 --- a/packages/app/src/lib/chart-utils.test.ts +++ b/packages/app/src/lib/chart-utils.test.ts @@ -167,6 +167,8 @@ function fullPt( inputTputY?: number; inputTputPerMwY?: number; outputTputPerMwY?: number; + outputTputPerTotalGpuY?: number; + outputTputPerTotalMwY?: number; }, ): InferenceData { return { @@ -209,6 +211,12 @@ function fullPt( ...(vals.outputTputPerMwY === undefined ? {} : { outputTputPerMw: { y: vals.outputTputPerMwY, roof: false } }), + ...(vals.outputTputPerTotalGpuY === undefined + ? {} + : { outputTputPerTotalGpu: { y: vals.outputTputPerTotalGpuY, roof: false } }), + ...(vals.outputTputPerTotalMwY === undefined + ? {} + : { outputTputPerTotalMw: { y: vals.outputTputPerTotalMwY, roof: false } }), } as InferenceData; } @@ -1064,6 +1072,45 @@ describe('createChartDataPoint', () => { expect(point.outputTputPerGpu).toBeUndefined(); }); + it('sets outputTputPerTotalGpu equal to outputTputPerGpu for non-disagg entries', () => { + const e = entry({ disagg: false, output_tput_per_gpu: 800 }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.outputTputPerTotalGpu).toEqual({ y: 800, roof: false }); + }); + + it('normalizes outputTputPerTotalGpu over prefill+decode GPUs for disagg entries', () => { + const e = entry({ + disagg: true, + num_prefill_gpu: 4, + num_decode_gpu: 2, + output_tput_per_gpu: 900, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + // (900 tok/s/decode-gpu * 2 decode GPUs) / (4 + 2 total GPUs) + expect(point.outputTputPerTotalGpu).toEqual({ y: 300, roof: false }); + }); + + it('omits total-GPU output metrics when output_tput_per_gpu is 0', () => { + const e = entry({ output_tput_per_gpu: 0 }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + expect(point.outputTputPerTotalGpu).toBeUndefined(); + expect(point.outputTputPerTotalMw).toBeUndefined(); + }); + + it('computes outputTputPerTotalMw from total-GPU output throughput and hardware power', () => { + const e = entry({ + disagg: true, + num_prefill_gpu: 4, + num_decode_gpu: 2, + output_tput_per_gpu: 900, + }); + const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); + // outputTputPerTotalGpu = 300, then (300 * 1000) / 700 (mocked power) + expect(point.outputTputPerTotalMw).toBeDefined(); + expect(point.outputTputPerTotalMw!.y).toBeCloseTo((300 * 1000) / 700, 5); + expect(point.outputTputPerTotalMw!.roof).toBe(false); + }); + it('sets inputTputPerGpu when input_tput_per_gpu > 0', () => { const e = entry({ input_tput_per_gpu: 300 }); const point = createChartDataPoint('2025-01-01', e, 'median_e2el', 'tput_per_gpu', 'h100'); @@ -1735,6 +1782,61 @@ describe('markRooflinePoints energy and output fields', () => { expect(m3.outputTputPerGpu!.roof).toBe(false); }); + it('marks outputTputPerTotalGpu.roof for points on the total-GPU output throughput roofline', () => { + const chartDef = { + chartType: 'e2e', + heading: 'Test', + x: 'median_e2el', + x_label: 'E2E Latency', + y: 'tput_per_gpu', + y_outputTputPerTotalGpu: 'outputTputPerTotalGpu.y', + y_outputTputPerTotalGpu_roofline: 'upper_right', + } as any; + + const p1 = fullPt(1, 'h100', { tpPerGpuY: 50, outputTputPerTotalGpuY: 200 }); + const p2 = fullPt(2, 'h100', { tpPerGpuY: 80, outputTputPerTotalGpuY: 500 }); + const p3 = fullPt(3, 'h100', { tpPerGpuY: 60, outputTputPerTotalGpuY: 350 }); + + const groupedData = { h100: [p1, p2, p3] }; + const rooflines = computeAllRooflines(groupedData, chartDef); + const marked = markRooflinePoints(groupedData, rooflines, chartDef); + + // upper_right: p1(1,200) and p2(2,500) on front; p3(3,350) < 500 so not on front + const m1 = marked.find((p) => p.x === 1)!; + const m2 = marked.find((p) => p.x === 2)!; + const m3 = marked.find((p) => p.x === 3)!; + expect(m1.outputTputPerTotalGpu!.roof).toBe(true); + expect(m2.outputTputPerTotalGpu!.roof).toBe(true); + expect(m3.outputTputPerTotalGpu!.roof).toBe(false); + }); + + it('marks outputTputPerTotalMw.roof for points on the total-GPU per-MW roofline', () => { + const chartDef = { + chartType: 'e2e', + heading: 'Test', + x: 'median_e2el', + x_label: 'E2E Latency', + y: 'tput_per_gpu', + y_outputTputPerTotalMw: 'outputTputPerTotalMw.y', + y_outputTputPerTotalMw_roofline: 'upper_right', + } as any; + + const p1 = fullPt(1, 'h100', { tpPerGpuY: 50, outputTputPerTotalMwY: 100 }); + const p2 = fullPt(2, 'h100', { tpPerGpuY: 80, outputTputPerTotalMwY: 400 }); + const p3 = fullPt(3, 'h100', { tpPerGpuY: 60, outputTputPerTotalMwY: 250 }); + + const groupedData = { h100: [p1, p2, p3] }; + const rooflines = computeAllRooflines(groupedData, chartDef); + const marked = markRooflinePoints(groupedData, rooflines, chartDef); + + const m1 = marked.find((p) => p.x === 1)!; + const m2 = marked.find((p) => p.x === 2)!; + const m3 = marked.find((p) => p.x === 3)!; + expect(m1.outputTputPerTotalMw!.roof).toBe(true); + expect(m2.outputTputPerTotalMw!.roof).toBe(true); + expect(m3.outputTputPerTotalMw!.roof).toBe(false); + }); + it('marks inputTputPerGpu.roof for points on the input throughput roofline', () => { const chartDef = { chartType: 'e2e', diff --git a/packages/app/src/lib/chart-utils.ts b/packages/app/src/lib/chart-utils.ts index d913a8cf..c79d75d5 100644 --- a/packages/app/src/lib/chart-utils.ts +++ b/packages/app/src/lib/chart-utils.ts @@ -168,9 +168,11 @@ export const Y_AXIS_METRICS = [ 'y_tpPerGpu', 'y_inputTputPerGpu', 'y_outputTputPerGpu', + 'y_outputTputPerTotalGpu', 'y_tpPerMw', 'y_inputTputPerMw', 'y_outputTputPerMw', + 'y_outputTputPerTotalMw', 'y_costh', 'y_costn', 'y_costr', @@ -310,6 +312,16 @@ export function createChartDataPoint( const outputTputPerGpu = entry.output_tput_per_gpu ?? 0; const inputTputPerGpu = entry.input_tput_per_gpu ?? 0; + // Output throughput normalized over the TOTAL GPU count. Disagg configs report + // output_tput_per_gpu per decode GPU only, so fold the prefill pool back in: + // (tok/s/decode-gpu * num_decode) / (num_prefill + num_decode). For aggregated + // configs every GPU decodes, so the value equals output_tput_per_gpu as-is. + const totalGpuCount = entry.num_prefill_gpu + entry.num_decode_gpu; + const outputTputPerTotalGpu = + entry.disagg && totalGpuCount > 0 + ? (outputTputPerGpu * entry.num_decode_gpu) / totalGpuCount + : outputTputPerGpu; + const tokensPerHour = (tputPerGpu * 3600) / 1000000; const outputTokensPerHour = (outputTputPerGpu * 3600) / 1000000; const inputTokensPerHour = (inputTputPerGpu * 3600) / 1000000; @@ -352,6 +364,9 @@ export function createChartDataPoint( // Roofline metric fields tpPerGpu: { y: tputPerGpu, roof: false }, ...(outputTputPerGpu ? { outputTputPerGpu: { y: outputTputPerGpu, roof: false } } : {}), + ...(outputTputPerGpu + ? { outputTputPerTotalGpu: { y: outputTputPerTotalGpu, roof: false } } + : {}), ...(inputTputPerGpu ? { inputTputPerGpu: { y: inputTputPerGpu, roof: false } } : {}), tpPerMw: { y: (tputPerGpu * 1000) / hardwarePower, roof: false }, ...(inputTputPerGpu @@ -370,6 +385,14 @@ export function createChartDataPoint( }, } : {}), + ...(outputTputPerGpu + ? { + outputTputPerTotalMw: { + y: hardwarePower ? (outputTputPerTotalGpu * 1000) / hardwarePower : 0, + roof: false, + }, + } + : {}), // Cost fields (combined throughput) costh: { @@ -627,10 +650,12 @@ export const calculateRoofline = ( | keyof InferenceData | `tpPerGpu.y` | `outputTputPerGpu.y` + | `outputTputPerTotalGpu.y` | `inputTputPerGpu.y` | `tpPerMw.y` | `inputTputPerMw.y` | `outputTputPerMw.y` + | `outputTputPerTotalMw.y` | `costh.y` | `costn.y` | `costr.y` @@ -705,10 +730,12 @@ export function computeAllRooflines( | keyof InferenceData | `tpPerGpu.y` | `outputTputPerGpu.y` + | `outputTputPerTotalGpu.y` | `inputTputPerGpu.y` | `tpPerMw.y` | `inputTputPerMw.y` | `outputTputPerMw.y` + | `outputTputPerTotalMw.y` | `costh.y` | `costn.y` | `costr.y` @@ -752,12 +779,16 @@ export function markRooflinePoints( if (newPoint.outputTputPerGpu) { newPoint.outputTputPerGpu.roof = false; } + if (newPoint.outputTputPerTotalGpu) { + newPoint.outputTputPerTotalGpu.roof = false; + } if (newPoint.inputTputPerGpu) { newPoint.inputTputPerGpu.roof = false; } newPoint.tpPerMw.roof = false; if (newPoint.inputTputPerMw) newPoint.inputTputPerMw.roof = false; if (newPoint.outputTputPerMw) newPoint.outputTputPerMw.roof = false; + if (newPoint.outputTputPerTotalMw) newPoint.outputTputPerTotalMw.roof = false; newPoint.costh.roof = false; newPoint.costn.roof = false; newPoint.costr.roof = false; @@ -801,6 +832,10 @@ export function markRooflinePoints( if (newPoint.outputTputPerGpu) { newPoint.outputTputPerGpu.roof = onCurrentRoofline; } + } else if (chartDefYKey === 'y_outputTputPerTotalGpu') { + if (newPoint.outputTputPerTotalGpu) { + newPoint.outputTputPerTotalGpu.roof = onCurrentRoofline; + } } else if (chartDefYKey === 'y_inputTputPerGpu') { if (newPoint.inputTputPerGpu) { newPoint.inputTputPerGpu.roof = onCurrentRoofline; @@ -811,6 +846,8 @@ export function markRooflinePoints( if (newPoint.inputTputPerMw) newPoint.inputTputPerMw.roof = onCurrentRoofline; } else if (chartDefYKey === 'y_outputTputPerMw') { if (newPoint.outputTputPerMw) newPoint.outputTputPerMw.roof = onCurrentRoofline; + } else if (chartDefYKey === 'y_outputTputPerTotalMw') { + if (newPoint.outputTputPerTotalMw) newPoint.outputTputPerTotalMw.roof = onCurrentRoofline; } else if (chartDefYKey === 'y_costh') { newPoint.costh.roof = onCurrentRoofline; } else if (chartDefYKey === 'y_costn') {