fix(scripts): preserve manual NOTES block in incremental report generator (#1119)

* fix(scripts): preserve manual NOTES block in incremental report generator

scripts/update-incremental-report.ts regenerates the report from scratch
on every release. PR #1008 added a manual <!-- NOTES_START --> ...
<!-- NOTES_END --> block explaining why 3.9.5 had null build metrics,
but the 3.9.6 release silently dropped it because the script had no
awareness of the sentinel pair. The version-history table now jumps
3.9.4 → 3.9.6 with no explanation.

Extract any NOTES block from the existing file (mirroring how
INCREMENTAL_BENCHMARK_DATA is preserved) and re-emit it just before
the data comment in the regenerated markdown. Restore the lost 3.9.5
note and add a regression test gated on the env-overridable report path.

Closes #1042

* fix(scripts): preserve all NOTES blocks via matchAll (#1119)

If multiple NOTES_START/NOTES_END blocks are added to annotate different
anomalous releases, .match() silently dropped all but the first — the same
data-loss class this PR was created to prevent. Switch to .matchAll() and
join with blank lines so every block survives regeneration. Add a test
covering two distinct NOTES blocks.

Author: carlos-alm

generated/benchmarks/INCREMENTAL-BENCHMARKS.md

@@ -88,6 +88,10 @@ Import resolution: native batch vs JS fallback throughput.
 | Per-import (JS) | 0ms |
 | Speedup ratio | 1.2x |
 
+<!-- NOTES_START -->
+**Note (3.9.5):** No build/rebuild metrics for this release (both engines null) — only import resolution data was collected. Both the WASM and native workers reached the 1-file rebuild phase and then hung past the benchmark's 10-minute per-engine timeout (see `scripts/lib/fork-engine.ts`), so each was killed (`SIGKILL`) before returning results. Import resolution is unaffected because it runs in the parent process and doesn't depend on the full build. 3.9.5 is consequently absent from the top-level version-history comparison table since there are no build-time figures to compare against prior releases. The workflow run is [here](https://github.com/optave/ops-codegraph-tool/actions/runs/24863501577); the root cause will be investigated and the numbers backfilled in a follow-up if possible.
+<!-- NOTES_END -->
+
 <!-- INCREMENTAL_BENCHMARK_DATA
 [
   {

scripts/update-incremental-report.ts

@@ -27,10 +27,13 @@ if (arg) {
 const entry = JSON.parse(jsonText);
 
 // ── Paths ────────────────────────────────────────────────────────────────
-const reportPath = path.join(root, 'generated', 'benchmarks', 'INCREMENTAL-BENCHMARKS.md');
+const reportPath =
+	process.env.CODEGRAPH_INCREMENTAL_REPORT_PATH ??
+	path.join(root, 'generated', 'benchmarks', 'INCREMENTAL-BENCHMARKS.md');
 
-// ── Load existing history ────────────────────────────────────────────────
+// ── Load existing history + manual NOTES block ───────────────────────────
 let history = [];
+let notesBlock = '';
 if (fs.existsSync(reportPath)) {
 	const content = fs.readFileSync(reportPath, 'utf8');
 	const match = content.match(/<!--\s*INCREMENTAL_BENCHMARK_DATA\s*([\s\S]*?)\s*-->/);
@@ -41,6 +44,13 @@ if (fs.existsSync(reportPath)) {
 			/* start fresh if corrupt */
 		}
 	}
+	// Use matchAll so multiple NOTES blocks (annotating different anomalous releases)
+	// are all preserved. The exact data-loss bug this fix targets stemmed from silently
+	// dropping a NOTES block; we must not reintroduce that failure mode for additional blocks.
+	const notesMatches = content.matchAll(
+		/<!--\s*NOTES_START\s*-->[\s\S]*?<!--\s*NOTES_END\s*-->/g,
+	);
+	notesBlock = Array.from(notesMatches, (m) => m[0]).join('\n\n');
 }
 
 // Add new entry — dev entries are rolling, releases replace dev
@@ -155,6 +165,8 @@ if (r.nativeBatchMs != null && r.jsFallbackMs > 0) {
 }
 md += '\n';
 
+if (notesBlock) md += `${notesBlock}\n\n`;
+
 md += `<!-- INCREMENTAL_BENCHMARK_DATA\n${JSON.stringify(history, null, 2)}\n-->\n`;
 
 fs.mkdirSync(path.dirname(reportPath), { recursive: true });

tests/unit/update-incremental-report.test.ts

@@ -0,0 +1,136 @@
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(__dirname, '..', '..');
+const scriptPath = path.join(repoRoot, 'scripts', 'update-incremental-report.ts');
+
+// --experimental-strip-types works in Node 22.6+ through current 24.x; the
+// renamed --strip-types was added then removed again across 24.x minor
+// versions, so prefer the experimental name for compatibility.
+const stripFlag = '--experimental-strip-types';
+
+const SAMPLE_ENTRY = {
+  version: '9.9.9',
+  date: '2026-05-14',
+  files: 100,
+  wasm: { fullBuildMs: 1000, noopRebuildMs: 10, oneFileRebuildMs: 50 },
+  native: { fullBuildMs: 500, noopRebuildMs: 5, oneFileRebuildMs: 25 },
+  resolve: {
+    imports: 200,
+    nativeBatchMs: 2,
+    jsFallbackMs: 4,
+    perImportNativeMs: 0,
+    perImportJsMs: 0,
+  },
+};
+
+let tmpDir: string;
+let reportPath: string;
+let entryPath: string;
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-incr-report-'));
+  reportPath = path.join(tmpDir, 'INCREMENTAL-BENCHMARKS.md');
+  entryPath = path.join(tmpDir, 'entry.json');
+  fs.writeFileSync(entryPath, JSON.stringify(SAMPLE_ENTRY));
+});
+
+afterEach(() => {
+  if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function runScript() {
+  execFileSync('node', [stripFlag, scriptPath, entryPath], {
+    env: { ...process.env, CODEGRAPH_INCREMENTAL_REPORT_PATH: reportPath },
+    stdio: 'pipe',
+  });
+}
+
+describe('update-incremental-report script', () => {
+  it('preserves a manual NOTES_START/NOTES_END block across regeneration', () => {
+    const NOTES = `<!-- NOTES_START -->
+**Note (9.9.8):** Workers hung past the 10-minute timeout and were SIGKILL'd.
+<!-- NOTES_END -->`;
+    const initial = `# Codegraph Incremental Build Benchmarks
+
+${NOTES}
+
+<!-- INCREMENTAL_BENCHMARK_DATA
+[
+  {
+    "version": "9.9.8",
+    "date": "2026-05-01",
+    "files": 99,
+    "wasm": null,
+    "native": null,
+    "resolve": { "imports": 100, "nativeBatchMs": 1, "jsFallbackMs": 2, "perImportNativeMs": 0, "perImportJsMs": 0 }
+  }
+]
+-->
+`;
+    fs.writeFileSync(reportPath, initial);
+
+    runScript();
+
+    const out = fs.readFileSync(reportPath, 'utf8');
+    expect(out).toContain('<!-- NOTES_START -->');
+    expect(out).toContain('<!-- NOTES_END -->');
+    expect(out).toContain("Workers hung past the 10-minute timeout and were SIGKILL'd");
+    // Notes should appear before the data comment, after the latest summary
+    expect(out.indexOf('<!-- NOTES_START -->')).toBeLessThan(
+      out.indexOf('<!-- INCREMENTAL_BENCHMARK_DATA'),
+    );
+  });
+
+  it('preserves multiple NOTES_START/NOTES_END blocks across regeneration', () => {
+    const NOTES_A = `<!-- NOTES_START -->
+**Note (9.9.8):** Workers hung past the 10-minute timeout and were SIGKILL'd.
+<!-- NOTES_END -->`;
+    const NOTES_B = `<!-- NOTES_START -->
+**Note (9.9.7):** Build artifact corrupted during upload, metrics re-run manually.
+<!-- NOTES_END -->`;
+    const initial = `# Codegraph Incremental Build Benchmarks
+
+${NOTES_A}
+
+${NOTES_B}
+
+<!-- INCREMENTAL_BENCHMARK_DATA
+[]
+-->
+`;
+    fs.writeFileSync(reportPath, initial);
+
+    runScript();
+
+    const out = fs.readFileSync(reportPath, 'utf8');
+    // Both notes must survive regeneration — single-block preservation would
+    // silently drop the second block (the same data-loss class this PR fixes).
+    expect(out).toContain("Workers hung past the 10-minute timeout and were SIGKILL'd");
+    expect(out).toContain('Build artifact corrupted during upload, metrics re-run manually');
+    // Each block keeps its own pair of delimiters.
+    expect(out.match(/<!--\s*NOTES_START\s*-->/g)?.length).toBe(2);
+    expect(out.match(/<!--\s*NOTES_END\s*-->/g)?.length).toBe(2);
+  });
+
+  it('does not invent a NOTES block when none was present', () => {
+    const initial = `# Codegraph Incremental Build Benchmarks
+
+<!-- INCREMENTAL_BENCHMARK_DATA
+[]
+-->
+`;
+    fs.writeFileSync(reportPath, initial);
+
+    runScript();
+
+    const out = fs.readFileSync(reportPath, 'utf8');
+    expect(out).not.toContain('NOTES_START');
+    expect(out).not.toContain('NOTES_END');
+  });
+});