test(h2-transport): comprehensive unit, integration, regression, and load tests

.github/workflows/h2-loadtest.yml

@@ -0,0 +1,36 @@
+name: H2 Load Tests
+
+on:
+  schedule:
+    - cron: '0 6 * * *'  # 06:00 UTC daily
+  workflow_dispatch:
+
+jobs:
+  loadtest:
+    name: h2-transport load tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: runloopai/checkout@main
+
+      - name: Set up Node
+        uses: runloopai/setup-node@main
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: yarn --frozen-lockfile
+
+      - name: Connect to Tailscale
+        uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2
+        with:
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:ci
+          hostname: github-ci-${{ github.run_id }}-loadtest
+
+      - name: Run load tests and push to Loki
+        env:
+          LOKI_URL: https://dev-loki
+        run: npx tsx loadtest/push-to-loki.ts

loadtest/README.md

@@ -0,0 +1,62 @@
+# h2-transport load tests
+
+Standalone scripts that exercise `src/lib/h2-transport/` under load. Each boots
+an in-process h2 server and runs the client until completion or a fixed
+duration. Not part of `npm test`; run on demand.
+
+## Scripts
+
+| Script | Purpose |
+|---|---|
+| `h2-multiplex.ts` | Fire N concurrent requests, report throughput + p50/p95/p99. |
+| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time. |
+| `h2-leak.ts` | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection. |
+| `h2-chaos.ts` | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. |
+| `h2load.sh` | Wraps `nghttp2`'s `h2load` against a long-running test server. |
+| `sse-test.ts` | Pre-existing manual SSE round-trip. |
+| `push-to-loki.ts` | Runs all four timed tests and pushes results to Loki for Grafana. |
+
+## Automated daily runs
+
+The `.github/workflows/h2-loadtest.yml` workflow fires at 06:00 UTC every day.
+It connects to the dev Tailscale network and runs `push-to-loki.ts`, which
+executes each script with CI-appropriate durations (multiplex N=1000,
+chaos 30 s, leak 120 s) and posts one structured JSON log line per test to
+Loki under `{job="h2-loadtest", test="<name>"}`.
+
+Results are visible in the **H2 Transport Load Tests** Grafana dashboard on
+dev-grafana. Each panel uses `last_over_time(...[1d])` to produce one data
+point per daily run, so regressions show up as step changes on the trend lines.
+
+To push results from a local run (requires Tailscale + access to dev-loki):
+
+```sh
+LOKI_URL=https://dev-loki npx tsx loadtest/push-to-loki.ts
+```
+
+## Run
+
+```sh
+npx tsx loadtest/h2-multiplex.ts          # default N=1000
+npx tsx loadtest/h2-multiplex.ts 10000
+npx tsx loadtest/h2-pool-growth.ts
+npx tsx loadtest/h2-leak.ts 600           # 10 min soak
+npx tsx loadtest/h2-chaos.ts 60
+./loadtest/h2load.sh 100000 100 32        # needs nghttp2 in $PATH
+```
+
+For the leak test, run with `node --expose-gc $(which npx) tsx loadtest/h2-leak.ts`
+so the periodic `global.gc()` calls fire — otherwise heap-growth signal is
+noisy with retained but reclaimable buffers.
+
+## Reading the output
+
+Each script prints a single JSON object at the end. Useful invariants:
+
+- `failures === 0` for `h2-multiplex` and `h2-pool-growth`.
+- `heapGrowthMB < 50` for `h2-leak`.
+- `getSuccessRate > 0.99` for `h2-chaos` once retries land; today we only
+  assert `> 0.50` because POST requests are not retried on session loss.
+
+Throughput numbers depend on the host and should be compared like-for-like,
+not against an absolute target.

loadtest/h2-chaos.ts

@@ -0,0 +1,181 @@
+/**
+ * Chaos test — server randomly drops sockets, sends RST_STREAM, GOAWAY, or
+ * delays headers. Asserts the client survives.
+ *
+ * Run: `npx tsx loadtest/h2-chaos.ts [durationSeconds=60]`
+ */
+import http2 from 'node:http2';
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createH2Fetch } from '../src/lib/h2-transport/index';
+
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+
+function makeCerts() {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'h2-chaos-'));
+  const key = path.join(tmp, 'key.pem');
+  const cert = path.join(tmp, 'cert.pem');
+  execFileSync(
+    'openssl',
+    [
+      'req',
+      '-x509',
+      '-newkey',
+      'rsa:2048',
+      '-keyout',
+      key,
+      '-out',
+      cert,
+      '-days',
+      '1',
+      '-nodes',
+      '-subj',
+      '/CN=localhost',
+    ],
+    { stdio: ['ignore', 'ignore', 'ignore'] },
+  );
+  return { key: fs.readFileSync(key), cert: fs.readFileSync(cert), tmp };
+}
+
+async function startServer(seed: number) {
+  const { key, cert, tmp } = makeCerts();
+  let rng = seed >>> 0;
+  const next = () => {
+    rng = (rng * 1664525 + 1013904223) >>> 0;
+    return rng / 0xffffffff;
+  };
+  const server = http2.createSecureServer({ key, cert });
+  server.on('stream', (stream) => {
+    stream.on('error', () => {});
+    const r = next();
+    if (r < 0.1) {
+      try {
+        stream.session?.socket?.destroy();
+      } catch {}
+      return;
+    }
+    if (r < 0.2) {
+      try {
+        stream.close(0x8);
+      } catch {}
+      return;
+    }
+    if (r < 0.21) {
+      stream.respond({ ':status': 200 });
+      stream.end('ok');
+      try {
+        stream.session?.goaway();
+      } catch {}
+      return;
+    }
+    const delay = r < 0.4 ? Math.floor(next() * 200) : 0;
+    setTimeout(() => {
+      if (stream.destroyed) return;
+      stream.respond({ ':status': 200 });
+      stream.end('ok');
+    }, delay);
+  });
+  return new Promise<{ port: number; close: () => void }>((resolve) => {
+    server.listen(0, () => {
+      resolve({
+        port: (server.address() as any).port,
+        close: () => {
+          server.close();
+          fs.rmSync(tmp, { recursive: true, force: true });
+        },
+      });
+    });
+  });
+}
+
+async function main() {
+  const durationSec = Number(process.argv[2] ?? 60);
+  if (!Number.isFinite(durationSec) || durationSec <= 0) {
+    console.error(`invalid duration: ${process.argv[2]} (must be a positive number of seconds)`);
+    process.exit(2);
+  }
+  const server = await startServer(42);
+  const fetch = createH2Fetch({
+    minConnections: 4,
+    maxConnections: 20,
+    connectTimeout: 5_000,
+    tlsOptions: { rejectUnauthorized: false },
+  });
+
+  const end = Date.now() + durationSec * 1000;
+  let getOk = 0,
+    getFail = 0,
+    postOk = 0,
+    postFail = 0;
+
+  async function get() {
+    try {
+      const r = (await fetch(`https://localhost:${server.port}/x`, { method: 'GET' } as any)) as any;
+      await r.text();
+      getOk++;
+    } catch {
+      getFail++;
+    }
+  }
+  async function post() {
+    try {
+      const r = (await fetch(`https://localhost:${server.port}/x`, {
+        method: 'POST',
+        body: 'b',
+      } as any)) as any;
+      await r.text();
+      postOk++;
+    } catch {
+      postFail++;
+    }
+  }
+
+  while (Date.now() < end) {
+    await Promise.all([...Array.from({ length: 10 }, get), ...Array.from({ length: 5 }, post)]);
+  }
+
+  await fetch.close();
+  server.close();
+
+  const getTotal = getOk + getFail;
+  const postTotal = postOk + postFail;
+  if (getTotal === 0) {
+    console.error('no GET requests were issued — load harness produced zero coverage');
+    process.exit(1);
+  }
+  const getRate = getOk / getTotal;
+  const postClean = postFail === 0 || postOk > 0;
+
+  console.log(
+    JSON.stringify(
+      {
+        getOk,
+        getFail,
+        getTotal,
+        getSuccessRate: Number(getRate.toFixed(3)),
+        postOk,
+        postFail,
+        postTotal,
+        durationSec,
+      },
+      null,
+      2,
+    ),
+  );
+
+  if (getRate < 0.5) {
+    console.error('GET success rate too low');
+    process.exit(1);
+  }
+  if (!postClean) {
+    console.error('POST never succeeded');
+    process.exit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});