diff --git a/loadtest/.gitignore b/loadtest/.gitignore new file mode 100644 index 000000000..504afef81 --- /dev/null +++ b/loadtest/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/loadtest/README.md b/loadtest/README.md index 7ab30bce0..af65ec61b 100644 --- a/loadtest/README.md +++ b/loadtest/README.md @@ -6,15 +6,47 @@ duration. Not part of `npm test`; run on demand. ## Scripts -| Script | Purpose | -|---|---| -| `h2-multiplex.ts` | Fire N concurrent requests, report throughput + p50/p95/p99. | -| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time. | -| `h2-leak.ts` | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection. | -| `h2-chaos.ts` | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. | -| `h2load.sh` | Wraps `nghttp2`'s `h2load` against a long-running test server. | -| `sse-test.ts` | Pre-existing manual SSE round-trip. | -| `push-to-loki.ts` | Runs all four timed tests and pushes results to Loki for Grafana. | +| Script | Purpose | +| ------------------- | ----------------------------------------------------------------------------------- | +| `h2-multiplex.ts` | Fire N concurrent requests, report throughput + p50/p95/p99. | +| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time. | +| `h2-leak.ts` | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection. | +| `h2-chaos.ts` | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. | +| `h2load.sh` | Wraps `nghttp2`'s `h2load` against a long-running test server. | +| `sse-test.ts` | Pre-existing manual SSE round-trip. | +| `push-to-loki.ts` | Runs all four timed tests and pushes results to Loki for Grafana. | + +## End-to-end transport comparison (real API) + +Unlike the in-process scripts above, these hit a real Runloop API (`RUNLOOP_API_KEY`, +optional `RUNLOOP_BASE_URL`). They send a burst of `devboxes.create` calls against a +**deliberately nonexistent blueprint** (`bp_nonexistent_loadtest_00000`), so every +request fails fast server-side (HTTP `400`) and **no devboxes are created** — isolating +client + server _request handling_ from provisioning. + +| Script | Transport under test | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `loadtest.ts` | The **SDK** itself. `USE_HTTP2=0` → HTTP/1.1 (`node-fetch`); default / `USE_HTTP2=1` → HTTP/2 (native `node:http2`). Imports `../src/sdk.ts`, so it always runs against the SDK code in this checkout. | +| `undici-test.ts`, `undici-single-conn.ts`, `undici-debug.ts` | Raw [undici](https://github.com/nodejs/undici) HTTP/2 pool, bypassing the SDK. | +| `h2-test.ts`, `h2-single-conn.ts` | Raw `node:http2`, bypassing the SDK. | +| `raw-fetch-test.ts` | Raw `node:https` (HTTP/1.1 keep-alive agent). | +| `alpn-check.ts` | Confirms the origin negotiates `h2` via TLS ALPN. | + +The raw-transport probes compare undici's HTTP/2 multiplexing against `node:http2` +(and HTTP/1.1) directly — the comparison that motivated building the SDK's own +`node:http2` transport. They're kept so that comparison stays reproducible. + +```sh +# SDK: HTTP/2 (default) vs HTTP/1.1, 2000-request burst +source ~/env && REQUEST_COUNT=2000 npx tsx loadtest/loadtest.ts # HTTP/2 +source ~/env && REQUEST_COUNT=2000 USE_HTTP2=0 npx tsx loadtest/loadtest.ts # HTTP/1.1 + +# Raw undici vs node:http2 comparison (undici comes from loadtest/package.json) +cd loadtest && npm install && source ~/env && npx tsx undici-test.ts +``` + +HTTP/1.1 opens a socket per in-flight request; for large bursts raise the file-descriptor +limit (`ulimit -n 65536`) or keep `REQUEST_COUNT` under it. ## Automated daily runs diff --git a/loadtest/alpn-check.ts b/loadtest/alpn-check.ts new file mode 100644 index 000000000..678c83573 --- /dev/null +++ b/loadtest/alpn-check.ts @@ -0,0 +1,24 @@ +import tls from 'node:tls'; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const url = new URL(BASE_URL); + +console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`); + +const socket = tls.connect( + { + host: url.hostname, + port: parseInt(url.port || '443', 10), + ALPNProtocols: ['h2', 'http/1.1'], + servername: url.hostname, + }, + () => { + console.log(`Negotiated protocol: ${socket.alpnProtocol}`); + console.log(`TLS version: ${socket.getProtocol()}`); + socket.end(); + }, +); + +socket.on('error', (err) => { + console.error('TLS error:', err.message); +}); diff --git a/loadtest/h2-single-conn.ts b/loadtest/h2-single-conn.ts new file mode 100644 index 000000000..e563274e2 --- /dev/null +++ b/loadtest/h2-single-conn.ts @@ -0,0 +1,66 @@ +import http2 from 'node:http2'; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-h2s-0', + environment_variables: { TEST_VAR_1: 'value_one' }, + launch_parameters: { resource_size_request: 'SMALL' }, +}); + +function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = client.request({ + ':method': 'POST', + ':path': '/v1/devboxes', + 'content-type': 'application/json', + authorization: `Bearer ${API_KEY}`, + }); + req.on('response', (headers) => { + const status = headers[':status'] as number; + req.on('data', () => {}); + req.on('end', () => resolve({ latencyMs: performance.now() - start, status })); + }); + req.on('error', reject); + req.end(body); + }); +} + +async function main() { + const url = new URL(BASE_URL); + const client = http2.connect(url.origin); + await new Promise((resolve, reject) => { + client.on('connect', resolve); + client.on('error', reject); + }); + + const maxStreams = client.remoteSettings?.maxConcurrentStreams; + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams}`); + + // Warmup + const w = await sendRequest(client); + console.log(`Warmup: status=${w.status}, latency=${w.latencyMs.toFixed(0)}ms`); + + // Burst 50 requests on single warmed connection + const count = 50; + console.log(`\nBursting ${count} requests on 1 warmed connection...`); + const wallStart = performance.now(); + const results = await Promise.all(Array.from({ length: count }, () => sendRequest(client))); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log( + `Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`, + ); + + client.close(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/h2-test.ts b/loadtest/h2-test.ts new file mode 100644 index 000000000..5da82ed6c --- /dev/null +++ b/loadtest/h2-test.ts @@ -0,0 +1,113 @@ +import http2 from 'node:http2'; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? '10', 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-h2-0', + environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' }, + metadata: { test_run: 'h2', index: '0' }, + launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, +}); + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +function connectH2(origin: string): Promise { + return new Promise((resolve, reject) => { + const client = http2.connect(origin); + client.on('connect', () => resolve(client)); + client.on('error', reject); + }); +} + +function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = client.request({ + ':method': 'POST', + ':path': '/v1/devboxes', + 'content-type': 'application/json', + authorization: `Bearer ${API_KEY}`, + }); + req.on('response', (headers) => { + const status = headers[':status'] as number; + req.on('data', () => {}); + req.on('end', () => resolve({ latencyMs: performance.now() - start, status })); + }); + req.on('error', reject); + req.end(body); + }); +} + +async function main() { + if (!Number.isInteger(NUM_CONNECTIONS) || NUM_CONNECTIONS < 1) { + console.error(`NUM_CONNECTIONS must be a positive integer (got "${process.env.NUM_CONNECTIONS}")`); + process.exit(1); + } + + console.log(`HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`); + + const url = new URL(BASE_URL); + const clients = await Promise.all(Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin))); + + const maxStreams = clients[0].remoteSettings?.maxConcurrentStreams; + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? 'unknown'}`); + console.log(`${NUM_CONNECTIONS} connections established\n`); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, 2000); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => { + const client = clients[i % NUM_CONNECTIONS]; + return sendRequest(client).then((r) => { + completed++; + return r; + }); + }); + + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + clearInterval(progressTimer); + + for (const c of clients) c.close(); + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\n=== HTTP/2 Results ===`); + console.log(`Requests: ${REQUEST_COUNT}`); + console.log(`Connections: ${NUM_CONNECTIONS}`); + console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + if (latencies.length > 0) { + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + } + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/loadtest.ts b/loadtest/loadtest.ts new file mode 100644 index 000000000..3f90264e0 --- /dev/null +++ b/loadtest/loadtest.ts @@ -0,0 +1,158 @@ +// Import the in-repo SDK source directly (run via `npx tsx`) so the benchmark +// always exercises the exact code in this checkout — not a published build. +import { Runloop } from '../src/sdk.ts'; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '100000', 10); +const RUNLOOP_BASE_URL = process.env.RUNLOOP_BASE_URL; +// HTTP/2 is the SDK's default transport; set USE_HTTP2=0 to benchmark HTTP/1.1. +const USE_HTTP2 = (process.env.USE_HTTP2 ?? '1') === '1'; +const PROGRESS_INTERVAL_MS = 2000; + +function buildClient(): Runloop { + const opts: ConstructorParameters[0] = { + maxRetries: 0, + timeout: 120_000, + }; + if (RUNLOOP_BASE_URL) { + opts.baseURL = RUNLOOP_BASE_URL; + } + // Set explicitly: since HTTP/2 is the default, HTTP/1.1 must be opted into + // with `http2: false` (omitting it would now select HTTP/2). + (opts as any).http2 = USE_HTTP2; + return new Runloop(opts); +} + +interface RequestResult { + index: number; + latencyMs: number; + status: number | null; + error: string | null; +} + +async function sendRequest(client: Runloop, index: number, runId: string): Promise { + const start = performance.now(); + try { + await client.devboxes.create({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: `loadtest-${runId}-${index}`, + environment_variables: { + TEST_VAR_1: 'value_one', + TEST_VAR_2: 'value_two', + }, + metadata: { + test_run: runId, + index: String(index), + }, + launch_parameters: { + resource_size_request: 'SMALL', + keep_alive_time_seconds: 300, + }, + }); + return { + index, + latencyMs: performance.now() - start, + status: 200, + error: null, + }; + } catch (err: any) { + return { + index, + latencyMs: performance.now() - start, + status: err?.status ?? null, + error: err?.message ?? String(err), + }; + } +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +function printMetrics(results: RequestResult[], wallClockMs: number): void { + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + + const statusCounts = new Map(); + for (const r of results) { + const key = r.status != null ? String(r.status) : 'network_error'; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); + } + + console.log('\n=== Load Test Results ==='); + console.log(`Requests: ${results.length}`); + console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`); + console.log(''); + if (latencies.length > 0) { + console.log('Latency (ms):'); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(''); + } + console.log('Status codes:'); + for (const [status, count] of [...statusCounts.entries()].sort()) { + console.log(` ${status}: ${count}`); + } +} + +async function main(): Promise { + const fdLimit = await checkFileDescriptorLimit(); + if (!USE_HTTP2 && fdLimit < 10000) { + console.warn( + `\nWARNING: File descriptor limit is ${fdLimit}. For 100k HTTP/1.1 requests, run:\n` + + ` ulimit -n 65536\n` + + `Or use HTTP/2 multiplexing: USE_HTTP2=1\n`, + ); + } + + const client = buildClient(); + const runId = `run-${Date.now()}`; + + console.log(`Starting load test: ${REQUEST_COUNT} concurrent requests`); + console.log(`Run ID: ${runId}`); + console.log(`HTTP mode: ${USE_HTTP2 ? 'HTTP/2 (SDK native node:http2)' : 'HTTP/1.1 (node-fetch)'}`); + console.log(`Base URL: ${RUNLOOP_BASE_URL ?? '(SDK default)'}`); + console.log(`File descriptor limit: ${fdLimit}`); + console.log(''); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, PROGRESS_INTERVAL_MS); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => + sendRequest(client, i, runId).then((result) => { + completed++; + return result; + }), + ); + + const results = await Promise.all(promises); + + const wallClockMs = performance.now() - wallStart; + clearInterval(progressTimer); + + printMetrics(results, wallClockMs); +} + +async function checkFileDescriptorLimit(): Promise { + try { + const { execSync } = await import('child_process'); + return parseInt(execSync('ulimit -n', { encoding: 'utf-8' }).trim(), 10); + } catch { + return -1; + } +} + +main().catch((err) => { + console.error('Fatal error:', err); + process.exit(1); +}); diff --git a/loadtest/package.json b/loadtest/package.json new file mode 100644 index 000000000..aa8b2d361 --- /dev/null +++ b/loadtest/package.json @@ -0,0 +1,14 @@ +{ + "name": "runloop-loadtest", + "private": true, + "scripts": { + "test": "tsx loadtest.ts" + }, + "dependencies": { + "tsx": "^4.0.0", + "undici": "^7.26.0" + }, + "devDependencies": { + "@types/node": "^25.9.3" + } +} diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts new file mode 100644 index 000000000..396590b63 --- /dev/null +++ b/loadtest/raw-fetch-test.ts @@ -0,0 +1,93 @@ +import https from 'node:https'; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const agent = new https.Agent({ + keepAlive: true, + maxSockets: Infinity, + maxFreeSockets: 4096, +}); + +const body = JSON.stringify({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-raw-0', + environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' }, + metadata: { test_run: 'raw', index: '0' }, + launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, +}); + +function makeRequest(index: number): Promise<{ latencyMs: number; status: number | null }> { + const start = performance.now(); + return new Promise((resolve) => { + // Settle exactly once. A request/response error or a mid-stream abort resolves + // with status=null (a failure) rather than hanging or rejecting the whole batch. + let settled = false; + const done = (status: number | null) => { + if (settled) return; + settled = true; + resolve({ latencyMs: performance.now() - start, status }); + }; + const url = new URL('/v1/devboxes', BASE_URL); + const req = https.request( + url, + { + method: 'POST', + agent, + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${API_KEY}`, + 'content-length': Buffer.byteLength(body).toString(), + }, + }, + (res) => { + res.resume(); + res.on('end', () => done(res.statusCode ?? null)); + res.on('aborted', () => done(null)); + res.on('error', () => done(null)); + }, + ); + req.on('error', () => done(null)); + req.end(body); + }); +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +async function main() { + console.log(`Raw node:https test: ${REQUEST_COUNT} concurrent requests to ${BASE_URL}`); + + const wallStart = performance.now(); + const results = await Promise.all(Array.from({ length: REQUEST_COUNT }, (_, i) => makeRequest(i))); + const wallMs = performance.now() - wallStart; + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + const key = r.status != null ? String(r.status) : 'network_error'; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); + } + + console.log(`\nWall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + if (latencies.length > 0) { + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + } + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-debug.ts b/loadtest/undici-debug.ts new file mode 100644 index 000000000..17c387ebc --- /dev/null +++ b/loadtest/undici-debug.ts @@ -0,0 +1,72 @@ +import { Agent, fetch } from 'undici'; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-debug-0', + environment_variables: { TEST_VAR_1: 'value_one' }, + launch_parameters: { resource_size_request: 'SMALL' }, +}); + +async function main() { + console.log('=== undici HTTP/2 debug ===\n'); + + const dispatcher = new Agent({ + allowH2: true, + connections: 2, + pipelining: 10, + keepAliveTimeout: 600_000, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Single request to inspect protocol + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + + console.log('Status:', res.status); + console.log('HTTP version:', res.headers.get('x-http-version') ?? '(not reported)'); + console.log('\nResponse headers:'); + for (const [k, v] of res.headers) { + console.log(` ${k}: ${v}`); + } + await res.text(); + + // Quick 20-request burst to verify concurrency + console.log('\n--- 20-request burst (2 connections, pipelining=10) ---'); + const wallStart = performance.now(); + const promises = Array.from({ length: 20 }, async () => { + const start = performance.now(); + const r = await fetch(`${BASE_URL}/v1/devboxes`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await r.text(); + return { latencyMs: performance.now() - start, status: r.status }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + console.log(`Wall clock: ${wallMs.toFixed(0)}ms`); + console.log(`Throughput: ${(20 / (wallMs / 1000)).toFixed(1)} req/s`); + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`Latency: min=${lats[0].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await dispatcher.close(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-single-conn.ts b/loadtest/undici-single-conn.ts new file mode 100644 index 000000000..b85cc0fd5 --- /dev/null +++ b/loadtest/undici-single-conn.ts @@ -0,0 +1,114 @@ +import { Client, Agent, fetch } from 'undici'; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const API_KEY = process.env.RUNLOOP_API_KEY!; +const PIPELINING = parseInt(process.env.PIPELINING ?? '100', 10); + +const body = JSON.stringify({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-single-0', + environment_variables: { TEST_VAR_1: 'value_one' }, + launch_parameters: { resource_size_request: 'SMALL' }, +}); + +async function testClient() { + console.log(`\n--- undici.Client (single connection, pipelining=${PIPELINING}) ---`); + const url = new URL(BASE_URL); + const client = new Client(url.origin, { + allowH2: true, + pipelining: PIPELINING, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Warm up - single request to establish connection + const warmup = await client.request({ + method: 'POST', + path: '/v1/devboxes', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, + body, + }); + await warmup.body.text(); + console.log(`Warmup: status=${warmup.statusCode}`); + + // Burst 50 requests on single connection + const count = 50; + const wallStart = performance.now(); + const promises = Array.from({ length: count }, async () => { + const start = performance.now(); + const res = await client.request({ + method: 'POST', + path: '/v1/devboxes', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, + body, + }); + await res.body.text(); + return { latencyMs: performance.now() - start, status: res.statusCode }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log( + `Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`, + ); + + await client.close(); +} + +async function testAgent() { + console.log(`\n--- undici.Agent (connections=1, pipelining=${PIPELINING}) ---`); + const dispatcher = new Agent({ + allowH2: true, + connections: 1, + pipelining: PIPELINING, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Warm up + const warmup = await fetch(`${BASE_URL}/v1/devboxes`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await warmup.text(); + console.log(`Warmup: status=${warmup.status}`); + + // Burst 50 requests + const count = 50; + const wallStart = performance.now(); + const promises = Array.from({ length: count }, async () => { + const start = performance.now(); + const r = await fetch(`${BASE_URL}/v1/devboxes`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await r.text(); + return { latencyMs: performance.now() - start, status: r.status }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log( + `Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`, + ); + + await dispatcher.close(); +} + +async function main() { + await testClient(); + await testAgent(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-test.ts b/loadtest/undici-test.ts new file mode 100644 index 000000000..f0221180e --- /dev/null +++ b/loadtest/undici-test.ts @@ -0,0 +1,103 @@ +import { Agent, fetch } from 'undici'; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? '20', 10); +const PIPELINING = parseInt(process.env.PIPELINING ?? '128', 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-undici-0', + environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' }, + metadata: { test_run: 'undici', index: '0' }, + launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, +}); + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +async function main() { + const dispatcher = new Agent({ + allowH2: true, + connections: NUM_CONNECTIONS, + pipelining: PIPELINING, + keepAliveTimeout: 600_000, + keepAliveMaxTimeout: 600_000, + bodyTimeout: 0, + headersTimeout: 0, + }); + + console.log( + `undici direct test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections, pipelining=${PIPELINING}`, + ); + console.log(`Target: ${BASE_URL}`); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, 2000); + + const wallStart = performance.now(); + + // Collect per-request results (never reject): one transient network error must + // not fail-fast the whole batch and discard every other completed measurement. + const promises = Array.from({ length: REQUEST_COUNT }, async () => { + const start = performance.now(); + try { + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + await res.text(); + return { latencyMs: performance.now() - start, status: res.status as number | null }; + } catch { + return { latencyMs: performance.now() - start, status: null as number | null }; + } finally { + completed++; + } + }); + + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + clearInterval(progressTimer); + + await dispatcher.close(); + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + const key = r.status != null ? String(r.status) : 'network_error'; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); + } + + console.log(`\n=== undici Direct Results ===`); + console.log(`Requests: ${REQUEST_COUNT}`); + console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + if (latencies.length > 0) { + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + } + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +});