|
| 1 | +/** |
| 2 | + * Restore runtime component metadata that older app ingest versions dropped. |
| 3 | + * |
| 4 | + * The benchmark producer began emitting nested router/offload component objects |
| 5 | + * and `kv_p2p_transfer` on 2026-07-13. The raw GitHub Actions benchmark |
| 6 | + * artifacts remain authoritative, so this script downloads only the small |
| 7 | + * benchmark-result artifacts and merges those metadata keys into matching DB |
| 8 | + * rows. It never replaces measured metrics or topology/config data. |
| 9 | + * |
| 10 | + * Usage: |
| 11 | + * pnpm --filter @semianalysisai/inferencex-db db:backfill-runtime-metadata |
| 12 | + * pnpm --filter @semianalysisai/inferencex-db db:backfill-runtime-metadata --yes |
| 13 | + */ |
| 14 | + |
| 15 | +import fs from 'node:fs'; |
| 16 | +import os from 'node:os'; |
| 17 | +import path from 'node:path'; |
| 18 | + |
| 19 | +import { hasNoSslFlag } from './cli-utils.js'; |
| 20 | +import { mapBenchmarkRow } from './etl/benchmark-mapper.js'; |
| 21 | +import { createAdminSql, refreshLatestBenchmarks } from './etl/db-utils.js'; |
| 22 | +import { createSkipTracker } from './etl/skip-tracker.js'; |
| 23 | +import { downloadArtifact, listRunArtifacts } from './lib/github-artifacts.js'; |
| 24 | +import { confirmProceed, runBackfillMain } from './lib/backfill-runner.js'; |
| 25 | +import { |
| 26 | + repositoryFromRunUrl, |
| 27 | + selectBenchmarkArtifacts, |
| 28 | +} from './lib/runtime-metadata-artifacts.js'; |
| 29 | + |
| 30 | +const REPO = 'SemiAnalysisAI/InferenceX'; |
| 31 | +const FIRST_METADATA_DATE = '2026-07-14'; |
| 32 | +const RUNTIME_KEYS = [ |
| 33 | + 'kv_offloading', |
| 34 | + 'kv_offload_backend', |
| 35 | + 'kv_offload_backend_version', |
| 36 | + 'kv_p2p_transfer', |
| 37 | + 'router_name', |
| 38 | + 'router_version', |
| 39 | +] as const; |
| 40 | + |
| 41 | +const sql = createAdminSql({ noSsl: hasNoSslFlag(), max: 2, onnotice: () => {} }); |
| 42 | + |
| 43 | +function findJsonFiles(root: string): string[] { |
| 44 | + const files: string[] = []; |
| 45 | + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { |
| 46 | + const pathname = path.join(root, entry.name); |
| 47 | + if (entry.isDirectory()) files.push(...findJsonFiles(pathname)); |
| 48 | + else if (entry.isFile() && entry.name.endsWith('.json')) files.push(pathname); |
| 49 | + } |
| 50 | + return files; |
| 51 | +} |
| 52 | + |
| 53 | +async function backfillRawRow(githubRunId: number, rawRow: Record<string, unknown>) { |
| 54 | + const mapped = mapBenchmarkRow(rawRow, createSkipTracker()); |
| 55 | + if (!mapped) return 'unmapped' as const; |
| 56 | + |
| 57 | + const metadata = Object.fromEntries( |
| 58 | + RUNTIME_KEYS.flatMap((key) => |
| 59 | + mapped.metrics[key] === undefined ? [] : [[key, mapped.metrics[key]]], |
| 60 | + ), |
| 61 | + ); |
| 62 | + if (Object.keys(metadata).length === 0) return 'empty' as const; |
| 63 | + |
| 64 | + const c = mapped.config; |
| 65 | + const updated = await sql<{ id: number }[]>` |
| 66 | + update benchmark_results br |
| 67 | + set metrics = br.metrics || ${sql.json(metadata)} |
| 68 | + from workflow_runs wr, configs cfg |
| 69 | + where br.workflow_run_id = wr.id |
| 70 | + and br.config_id = cfg.id |
| 71 | + and wr.github_run_id = ${githubRunId} |
| 72 | + and cfg.hardware = ${c.hardware} |
| 73 | + and cfg.framework = ${c.framework} |
| 74 | + and cfg.model = ${c.model} |
| 75 | + and cfg.precision = ${c.precision} |
| 76 | + and cfg.spec_method = ${c.specMethod} |
| 77 | + and cfg.disagg = ${c.disagg} |
| 78 | + and cfg.is_multinode = ${c.isMultinode} |
| 79 | + and cfg.prefill_tp = ${c.prefillTp} |
| 80 | + and cfg.prefill_ep = ${c.prefillEp} |
| 81 | + and cfg.prefill_dp_attention = ${c.prefillDpAttn} |
| 82 | + and cfg.prefill_num_workers = ${c.prefillNumWorkers} |
| 83 | + and cfg.decode_tp = ${c.decodeTp} |
| 84 | + and cfg.decode_ep = ${c.decodeEp} |
| 85 | + and cfg.decode_dp_attention = ${c.decodeDpAttn} |
| 86 | + and cfg.decode_num_workers = ${c.decodeNumWorkers} |
| 87 | + and cfg.num_prefill_gpu = ${c.numPrefillGpu} |
| 88 | + and cfg.num_decode_gpu = ${c.numDecodeGpu} |
| 89 | + and br.benchmark_type = ${mapped.benchmarkType} |
| 90 | + and br.isl is not distinct from ${mapped.isl} |
| 91 | + and br.osl is not distinct from ${mapped.osl} |
| 92 | + and br.conc = ${mapped.conc} |
| 93 | + and br.offload_mode = ${mapped.offloadMode} |
| 94 | + returning br.id |
| 95 | + `; |
| 96 | + return updated.length > 0 ? ('updated' as const) : ('missing' as const); |
| 97 | +} |
| 98 | + |
| 99 | +async function main(): Promise<void> { |
| 100 | + console.log('=== backfill-runtime-metadata ==='); |
| 101 | + const runs = await sql<{ github_run_id: number; html_url: string | null }[]>` |
| 102 | + select distinct wr.github_run_id, wr.html_url |
| 103 | + from workflow_runs wr |
| 104 | + join benchmark_results br on br.workflow_run_id = wr.id |
| 105 | + where wr.date >= ${FIRST_METADATA_DATE}::date |
| 106 | + order by wr.github_run_id |
| 107 | + `; |
| 108 | + if (runs.length === 0) { |
| 109 | + console.log(' Nothing to do.'); |
| 110 | + return; |
| 111 | + } |
| 112 | + if (!(await confirmProceed(`${runs.length} workflow run(s) may contain runtime metadata.`))) { |
| 113 | + return; |
| 114 | + } |
| 115 | + |
| 116 | + let updated = 0; |
| 117 | + let missing = 0; |
| 118 | + let empty = 0; |
| 119 | + let unmapped = 0; |
| 120 | + for (const [index, run] of runs.entries()) { |
| 121 | + const runId = Number(run.github_run_id); |
| 122 | + const repository = repositoryFromRunUrl(run.html_url) ?? REPO; |
| 123 | + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), `runtime-metadata-${runId}-`)); |
| 124 | + try { |
| 125 | + const artifacts = selectBenchmarkArtifacts(listRunArtifacts(repository, String(runId))); |
| 126 | + let files = 0; |
| 127 | + for (const artifact of artifacts) { |
| 128 | + const artifactDir = downloadArtifact(artifact, tempDir); |
| 129 | + for (const file of findJsonFiles(artifactDir)) { |
| 130 | + files++; |
| 131 | + const parsed = JSON.parse(fs.readFileSync(file, 'utf8')) as unknown; |
| 132 | + const rawRows = Array.isArray(parsed) ? parsed : [parsed]; |
| 133 | + for (const raw of rawRows) { |
| 134 | + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) continue; |
| 135 | + const result = await backfillRawRow(runId, raw as Record<string, unknown>); |
| 136 | + if (result === 'updated') updated++; |
| 137 | + else if (result === 'missing') missing++; |
| 138 | + else if (result === 'empty') empty++; |
| 139 | + else unmapped++; |
| 140 | + } |
| 141 | + } |
| 142 | + } |
| 143 | + console.log( |
| 144 | + ` [${index + 1}/${runs.length}] ${repository} run ${runId}: ` + |
| 145 | + `${artifacts.length} artifact(s), ${files} file(s)`, |
| 146 | + ); |
| 147 | + } finally { |
| 148 | + fs.rmSync(tempDir, { recursive: true, force: true }); |
| 149 | + } |
| 150 | + } |
| 151 | + |
| 152 | + await refreshLatestBenchmarks(sql); |
| 153 | + console.log( |
| 154 | + ` Rows: ${updated} updated, ${empty} without runtime metadata, ` + |
| 155 | + `${unmapped} unmapped, ${missing} missing DB match`, |
| 156 | + ); |
| 157 | + if (missing > 0) process.exitCode = 1; |
| 158 | +} |
| 159 | + |
| 160 | +runBackfillMain('backfill-runtime-metadata', sql, main); |
0 commit comments