feat: add HTTP/2 load testing infrastructure (#798)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: Reflex <reflex@runloop.ai>

Author: 3 people

loadtest/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+package-lock.json

loadtest/README.md

@@ -6,15 +6,47 @@ duration. Not part of `npm test`; run on demand.
 
 ## Scripts
 
-| Script | Purpose |
-|---|---|
-| `h2-multiplex.ts` | Fire N concurrent requests, report throughput + p50/p95/p99. |
-| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time. |
-| `h2-leak.ts` | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection. |
-| `h2-chaos.ts` | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. |
-| `h2load.sh` | Wraps `nghttp2`'s `h2load` against a long-running test server. |
-| `sse-test.ts` | Pre-existing manual SSE round-trip. |
-| `push-to-loki.ts` | Runs all four timed tests and pushes results to Loki for Grafana. |
+| Script              | Purpose                                                                             |
+| ------------------- | ----------------------------------------------------------------------------------- |
+| `h2-multiplex.ts`   | Fire N concurrent requests, report throughput + p50/p95/p99.                        |
+| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time.                 |
+| `h2-leak.ts`        | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection.                  |
+| `h2-chaos.ts`       | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. |
+| `h2load.sh`         | Wraps `nghttp2`'s `h2load` against a long-running test server.                      |
+| `sse-test.ts`       | Pre-existing manual SSE round-trip.                                                 |
+| `push-to-loki.ts`   | Runs all four timed tests and pushes results to Loki for Grafana.                   |
+
+## End-to-end transport comparison (real API)
+
+Unlike the in-process scripts above, these hit a real Runloop API (`RUNLOOP_API_KEY`,
+optional `RUNLOOP_BASE_URL`). They send a burst of `devboxes.create` calls against a
+**deliberately nonexistent blueprint** (`bp_nonexistent_loadtest_00000`), so every
+request fails fast server-side (HTTP `400`) and **no devboxes are created** — isolating
+client + server _request handling_ from provisioning.
+
+| Script                                                       | Transport under test                                                                                                                                                                                   |
+| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `loadtest.ts`                                                | The **SDK** itself. `USE_HTTP2=0` → HTTP/1.1 (`node-fetch`); default / `USE_HTTP2=1` → HTTP/2 (native `node:http2`). Imports `../src/sdk.ts`, so it always runs against the SDK code in this checkout. |
+| `undici-test.ts`, `undici-single-conn.ts`, `undici-debug.ts` | Raw [undici](https://github.com/nodejs/undici) HTTP/2 pool, bypassing the SDK.                                                                                                                         |
+| `h2-test.ts`, `h2-single-conn.ts`                            | Raw `node:http2`, bypassing the SDK.                                                                                                                                                                   |
+| `raw-fetch-test.ts`                                          | Raw `node:https` (HTTP/1.1 keep-alive agent).                                                                                                                                                          |
+| `alpn-check.ts`                                              | Confirms the origin negotiates `h2` via TLS ALPN.                                                                                                                                                      |
+
+The raw-transport probes compare undici's HTTP/2 multiplexing against `node:http2`
+(and HTTP/1.1) directly — the comparison that motivated building the SDK's own
+`node:http2` transport. They're kept so that comparison stays reproducible.
+
+```sh
+# SDK: HTTP/2 (default) vs HTTP/1.1, 2000-request burst
+source ~/env && REQUEST_COUNT=2000 npx tsx loadtest/loadtest.ts                 # HTTP/2
+source ~/env && REQUEST_COUNT=2000 USE_HTTP2=0 npx tsx loadtest/loadtest.ts     # HTTP/1.1
+
+# Raw undici vs node:http2 comparison (undici comes from loadtest/package.json)
+cd loadtest && npm install && source ~/env && npx tsx undici-test.ts
+```
+
+HTTP/1.1 opens a socket per in-flight request; for large bursts raise the file-descriptor
+limit (`ulimit -n 65536`) or keep `REQUEST_COUNT` under it.
 
 ## Automated daily runs
 

loadtest/alpn-check.ts

@@ -0,0 +1,24 @@
+import tls from 'node:tls';
+
+const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro';
+const url = new URL(BASE_URL);
+
+console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`);
+
+const socket = tls.connect(
+  {
+    host: url.hostname,
+    port: parseInt(url.port || '443', 10),
+    ALPNProtocols: ['h2', 'http/1.1'],
+    servername: url.hostname,
+  },
+  () => {
+    console.log(`Negotiated protocol: ${socket.alpnProtocol}`);
+    console.log(`TLS version: ${socket.getProtocol()}`);
+    socket.end();
+  },
+);
+
+socket.on('error', (err) => {
+  console.error('TLS error:', err.message);
+});

loadtest/h2-single-conn.ts

@@ -0,0 +1,66 @@
+import http2 from 'node:http2';
+
+const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro';
+const API_KEY = process.env.RUNLOOP_API_KEY!;
+
+const body = JSON.stringify({
+  blueprint_id: 'bp_nonexistent_loadtest_00000',
+  name: 'loadtest-h2s-0',
+  environment_variables: { TEST_VAR_1: 'value_one' },
+  launch_parameters: { resource_size_request: 'SMALL' },
+});
+
+function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> {
+  return new Promise((resolve, reject) => {
+    const start = performance.now();
+    const req = client.request({
+      ':method': 'POST',
+      ':path': '/v1/devboxes',
+      'content-type': 'application/json',
+      authorization: `Bearer ${API_KEY}`,
+    });
+    req.on('response', (headers) => {
+      const status = headers[':status'] as number;
+      req.on('data', () => {});
+      req.on('end', () => resolve({ latencyMs: performance.now() - start, status }));
+    });
+    req.on('error', reject);
+    req.end(body);
+  });
+}
+
+async function main() {
+  const url = new URL(BASE_URL);
+  const client = http2.connect(url.origin);
+  await new Promise<void>((resolve, reject) => {
+    client.on('connect', resolve);
+    client.on('error', reject);
+  });
+
+  const maxStreams = client.remoteSettings?.maxConcurrentStreams;
+  console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams}`);
+
+  // Warmup
+  const w = await sendRequest(client);
+  console.log(`Warmup: status=${w.status}, latency=${w.latencyMs.toFixed(0)}ms`);
+
+  // Burst 50 requests on single warmed connection
+  const count = 50;
+  console.log(`\nBursting ${count} requests on 1 warmed connection...`);
+  const wallStart = performance.now();
+  const results = await Promise.all(Array.from({ length: count }, () => sendRequest(client)));
+  const wallMs = performance.now() - wallStart;
+
+  const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b);
+  console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`);
+  console.log(
+    `Latency: min=${lats[0].toFixed(0)}ms  p50=${lats[Math.floor(count / 2)].toFixed(0)}ms  max=${lats[lats.length - 1].toFixed(0)}ms`,
+  );
+
+  client.close();
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});

loadtest/h2-test.ts

@@ -0,0 +1,113 @@
+import http2 from 'node:http2';
+
+const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10);
+const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? '10', 10);
+const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro';
+const API_KEY = process.env.RUNLOOP_API_KEY!;
+
+const body = JSON.stringify({
+  blueprint_id: 'bp_nonexistent_loadtest_00000',
+  name: 'loadtest-h2-0',
+  environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' },
+  metadata: { test_run: 'h2', index: '0' },
+  launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 },
+});
+
+function percentile(sorted: number[], p: number): number {
+  const idx = Math.ceil((p / 100) * sorted.length) - 1;
+  return sorted[Math.max(0, idx)];
+}
+
+function connectH2(origin: string): Promise<http2.ClientHttp2Session> {
+  return new Promise((resolve, reject) => {
+    const client = http2.connect(origin);
+    client.on('connect', () => resolve(client));
+    client.on('error', reject);
+  });
+}
+
+function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> {
+  return new Promise((resolve, reject) => {
+    const start = performance.now();
+    const req = client.request({
+      ':method': 'POST',
+      ':path': '/v1/devboxes',
+      'content-type': 'application/json',
+      authorization: `Bearer ${API_KEY}`,
+    });
+    req.on('response', (headers) => {
+      const status = headers[':status'] as number;
+      req.on('data', () => {});
+      req.on('end', () => resolve({ latencyMs: performance.now() - start, status }));
+    });
+    req.on('error', reject);
+    req.end(body);
+  });
+}
+
+async function main() {
+  if (!Number.isInteger(NUM_CONNECTIONS) || NUM_CONNECTIONS < 1) {
+    console.error(`NUM_CONNECTIONS must be a positive integer (got "${process.env.NUM_CONNECTIONS}")`);
+    process.exit(1);
+  }
+
+  console.log(`HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`);
+
+  const url = new URL(BASE_URL);
+  const clients = await Promise.all(Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin)));
+
+  const maxStreams = clients[0].remoteSettings?.maxConcurrentStreams;
+  console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? 'unknown'}`);
+  console.log(`${NUM_CONNECTIONS} connections established\n`);
+
+  let completed = 0;
+  const progressTimer = setInterval(() => {
+    console.log(
+      `  progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`,
+    );
+  }, 2000);
+
+  const wallStart = performance.now();
+
+  const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => {
+    const client = clients[i % NUM_CONNECTIONS];
+    return sendRequest(client).then((r) => {
+      completed++;
+      return r;
+    });
+  });
+
+  const results = await Promise.all(promises);
+  const wallMs = performance.now() - wallStart;
+  clearInterval(progressTimer);
+
+  for (const c of clients) c.close();
+
+  const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b);
+  const statusCounts = new Map<number, number>();
+  for (const r of results) {
+    statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1);
+  }
+
+  console.log(`\n=== HTTP/2 Results ===`);
+  console.log(`Requests:    ${REQUEST_COUNT}`);
+  console.log(`Connections: ${NUM_CONNECTIONS}`);
+  console.log(`Wall clock:  ${(wallMs / 1000).toFixed(2)}s`);
+  console.log(`Throughput:  ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`);
+  if (latencies.length > 0) {
+    console.log(`\nLatency (ms):`);
+    console.log(`  min: ${latencies[0].toFixed(1)}`);
+    console.log(`  p50: ${percentile(latencies, 50).toFixed(1)}`);
+    console.log(`  p90: ${percentile(latencies, 90).toFixed(1)}`);
+    console.log(`  p95: ${percentile(latencies, 95).toFixed(1)}`);
+    console.log(`  p99: ${percentile(latencies, 99).toFixed(1)}`);
+    console.log(`  max: ${latencies[latencies.length - 1].toFixed(1)}`);
+  }
+  console.log(`\nStatus codes:`);
+  for (const [s, c] of [...statusCounts.entries()].sort()) console.log(`  ${s}: ${c}`);
+}
+
+main().catch((e) => {
+  console.error(e);
+  process.exit(1);
+});