feat: surface #7762 metrics in CSV + enable in dive workflow (#100)

Three follow-ups to make the next dive run informative:

1. Workflow now patches settings.json to set
   scalingDiveMetrics=true (added to core in #7762 but defaulting
   off there per project rule). Without this the three new metric
   rows the harness wants would never appear on /stats/prometheus.

2. CSV column rename: evloop_p95_ms -> evloop_p99_ms. prom-client's
   collectDefaultMetrics emits nodejs_eventloop_lag_p99_seconds
   (p50/p90/p99 — no p95), so the previous lookup was always empty.
   MD table header updated to match.

3. Two new curated CSV columns:
   - apply_mean_ms: from etherpad_changeset_apply_duration_seconds_sum
     and _count (histogram mean), converted to ms. Lets the dive
     attribute server-side latency to the apply path vs fan-out.
   - emits_new_changes: from etherpad_socket_emits_total{type=NEW_CHANGES}.
     Dominant fan-out cost; the column makes the batching lever's
     payoff visible.

Both new columns are populated only when the underlying metrics
exist on the SUT; older builds get empty cells (existing pattern).

48 tests green.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: JohnMcLear

.github/workflows/scaling-dive.yml

@@ -99,6 +99,13 @@ jobs:
             s!"points":[^,]*!"points": 1000!
           ' settings.json.template > settings.json
 
+          # Enable the scaling-dive metrics added in ether/etherpad#7762.
+          # The key isn't in settings.json.template yet (defaults to false in
+          # Settings.ts), so we inject it. Use node so JSON manipulation
+          # stays valid regardless of key ordering / whitespace.
+          node -e "const fs=require('fs'); const s=JSON.parse(fs.readFileSync('settings.json','utf8')); s.scalingDiveMetrics=true; fs.writeFileSync('settings.json', JSON.stringify(s, null, 2));"
+          echo 'enabled: scalingDiveMetrics = true'
+
           case "${{ matrix.lever }}" in
             baseline)
               echo "baseline — no further settings tweaks"

src/sim/reporter.ts

@@ -60,7 +60,14 @@ export class Reporter {
   }
 
   private toCsv(): string {
-    const header = 'step,p50,p95,p99,max,throughput_csps,cpu_user,evloop_p95_ms,rss_mb,users,errors,break';
+    // CSV column set is fixed. New gauges added via --scrape-keep are
+    // queryable from JSON only — that's intentional so the CSV schema
+    // stays stable across runs.
+    //
+    // apply_mean_ms and emits_new_changes are populated only when
+    // ether/etherpad's `scalingDiveMetrics` setting is true (registered
+    // since #7762). The harness tolerates them being absent.
+    const header = 'step,p50,p95,p99,max,throughput_csps,cpu_user,evloop_p99_ms,rss_mb,users,apply_mean_ms,emits_new_changes,errors,break';
     const fmt = (n: number | undefined): string =>
       n === undefined || !Number.isFinite(n) ? '' : String(n);
     const cell = (g: Record<string, number>, key: string): string => {
@@ -70,12 +77,21 @@ export class Reporter {
     const rows = this.steps.map((s) => {
       const g = s.snapshot.gauges;
       // CSV column → /stats/prometheus row mapping. Names come from prom-client's
-      // collectDefaultMetrics output that etherpad core actually emits.
+      // collectDefaultMetrics output that etherpad core actually emits, plus the
+      // three custom metrics added in ether/etherpad#7762.
       const rssBytes = g['process_resident_memory_bytes'];
       const rssMb = rssBytes !== undefined ? Math.round(rssBytes / 1_048_576) : undefined;
-      const evloopP95s = g['nodejs_eventloop_lag_p95_seconds'];
-      const evloopP95Ms = evloopP95s !== undefined ? Math.round(evloopP95s * 1000) : undefined;
+      // prom-client exposes _p50/_p90/_p99 (no p95). p99 is the closest tail metric.
+      const evloopP99s = g['nodejs_eventloop_lag_p99_seconds'];
+      const evloopP99Ms = evloopP99s !== undefined ? Math.round(evloopP99s * 1000) : undefined;
       const cpuUserS = g['process_cpu_user_seconds_total'];
+      // Histogram: mean = sum / count, converted to ms.
+      const applySum = g['etherpad_changeset_apply_duration_seconds_sum'];
+      const applyCount = g['etherpad_changeset_apply_duration_seconds_count'];
+      const applyMeanMs = applySum !== undefined && applyCount !== undefined && applyCount > 0
+        ? Math.round((applySum / applyCount) * 1000 * 100) / 100
+        : undefined;
+      const emitsNewChanges = g['etherpad_socket_emits_total{type=NEW_CHANGES}'];
       return [
         s.step,
         fmt(s.latencyMs.p50),
@@ -84,9 +100,11 @@ export class Reporter {
         fmt(s.latencyMs.max),
         fmt(s.throughputCsps),
         fmt(cpuUserS),
-        fmt(evloopP95Ms),
+        fmt(evloopP99Ms),
         fmt(rssMb),
         cell(g, 'etherpad_total_users'),
+        fmt(applyMeanMs),
+        fmt(emitsNewChanges),
         s.errors,
         s.breakageFlags.join('|'),
       ].join(',');
@@ -101,12 +119,12 @@ export class Reporter {
       `Run: ${m.runId}`,
       `SUT: ${m.sut.gitSha ?? '?'} (${m.sut.version ?? '?'}) on ${m.machine.cpus} · ${m.machine.totalMemMB} MB · node ${m.machine.node}`,
       '',
-      '| Step | p50 | p95 | p99 | EL p95 | CPU% | Errors | Break |',
+      '| Step | p50 | p95 | p99 | EL p99 | CPU% | Errors | Break |',
       '|---:|---:|---:|---:|---:|---:|---:|:---|',
     ];
     const rows = this.steps.map((s) => {
       const g = s.snapshot.gauges;
-      const elS = g['nodejs_eventloop_lag_p95_seconds'];
+      const elS = g['nodejs_eventloop_lag_p99_seconds'];
       const elMs = elS !== undefined ? Math.round(elS * 1000) : '';
       const cpuS = g['process_cpu_user_seconds_total'];
       const cpu = cpuS !== undefined ? cpuS.toFixed(2) : '';

tests/sim/reporter.test.ts

@@ -66,7 +66,7 @@ describe('Reporter CSV output', () => {
     const paths = await r.write();
     const csv = readFileSync(paths.csv, 'utf8');
     const lines = csv.trim().split('\n');
-    expect(lines[0]).toBe('step,p50,p95,p99,max,throughput_csps,cpu_user,evloop_p95_ms,rss_mb,users,errors,break');
+    expect(lines[0]).toBe('step,p50,p95,p99,max,throughput_csps,cpu_user,evloop_p99_ms,rss_mb,users,apply_mean_ms,emits_new_changes,errors,break');
     expect(lines[1]!.split(',')[0]).toBe('10');
     expect(lines[1]!.split(',')[1]).toBe('20');
     expect(lines[2]!.split(',')[0]).toBe('20');
@@ -81,11 +81,13 @@ describe('Reporter CSV output', () => {
     const paths = await r.write();
     const csv = readFileSync(paths.csv, 'utf8');
     const cols = csv.trim().split('\n')[1]!.split(',');
-    // step,p50,p95,p99,max,throughput_csps,<cpu_user>,<evloop_p95_ms>,<rss_mb>,<users>,errors,break
-    expect(cols[6]).toBe(''); // cpu_user missing
-    expect(cols[7]).toBe(''); // evloop missing
-    expect(cols[8]).toBe(''); // rss missing
-    expect(cols[9]).toBe(''); // users missing
+    // step,p50,p95,p99,max,throughput_csps,<cpu_user>,<evloop_p99_ms>,<rss_mb>,<users>,<apply_mean_ms>,<emits_new_changes>,errors,break
+    expect(cols[6]).toBe('');  // cpu_user missing
+    expect(cols[7]).toBe('');  // evloop missing
+    expect(cols[8]).toBe('');  // rss missing
+    expect(cols[9]).toBe('');  // users missing
+    expect(cols[10]).toBe(''); // apply_mean_ms missing
+    expect(cols[11]).toBe(''); // emits_new_changes missing
   });
 });