From 4939796f6a4e0731b158a3b6d831f4d98d0cdeab Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Thu, 11 Jun 2026 16:27:25 -0700 Subject: [PATCH 1/5] feat: add HTTP/2 load testing infrastructure Adds a loadtest/ directory with scripts for benchmarking HTTP/2 transport throughput against the Runloop API. Includes baselines for raw node:http2, undici, and the SDK client at various concurrency levels. Co-Authored-By: Claude Opus 4.6 --- loadtest/.gitignore | 2 + loadtest/alpn-check.ts | 24 +++++ loadtest/h2-single-conn.ts | 61 +++++++++++++ loadtest/h2-test.ts | 112 +++++++++++++++++++++++ loadtest/loadtest.ts | 159 +++++++++++++++++++++++++++++++++ loadtest/package.json | 16 ++++ loadtest/raw-fetch-test.ts | 84 +++++++++++++++++ loadtest/undici-debug.ts | 72 +++++++++++++++ loadtest/undici-single-conn.ts | 107 ++++++++++++++++++++++ loadtest/undici-test.ts | 93 +++++++++++++++++++ 10 files changed, 730 insertions(+) create mode 100644 loadtest/.gitignore create mode 100644 loadtest/alpn-check.ts create mode 100644 loadtest/h2-single-conn.ts create mode 100644 loadtest/h2-test.ts create mode 100644 loadtest/loadtest.ts create mode 100644 loadtest/package.json create mode 100644 loadtest/raw-fetch-test.ts create mode 100644 loadtest/undici-debug.ts create mode 100644 loadtest/undici-single-conn.ts create mode 100644 loadtest/undici-test.ts diff --git a/loadtest/.gitignore b/loadtest/.gitignore new file mode 100644 index 000000000..504afef81 --- /dev/null +++ b/loadtest/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/loadtest/alpn-check.ts b/loadtest/alpn-check.ts new file mode 100644 index 000000000..9a59813c1 --- /dev/null +++ b/loadtest/alpn-check.ts @@ -0,0 +1,24 @@ +import tls from "node:tls"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const url = new URL(BASE_URL); + +console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`); + +const socket = tls.connect( + { + host: url.hostname, + port: parseInt(url.port || "443", 10), + ALPNProtocols: ["h2", "http/1.1"], + servername: url.hostname, + }, + () => { + console.log(`Negotiated protocol: ${socket.alpnProtocol}`); + console.log(`TLS version: ${socket.getProtocol()}`); + socket.end(); + }, +); + +socket.on("error", (err) => { + console.error("TLS error:", err.message); +}); diff --git a/loadtest/h2-single-conn.ts b/loadtest/h2-single-conn.ts new file mode 100644 index 000000000..7ceb78ed1 --- /dev/null +++ b/loadtest/h2-single-conn.ts @@ -0,0 +1,61 @@ +import http2 from "node:http2"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-h2s-0", + environment_variables: { TEST_VAR_1: "value_one" }, + launch_parameters: { resource_size_request: "SMALL" }, +}); + +function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = client.request({ + ":method": "POST", + ":path": "/v1/devboxes", + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }); + req.on("response", (headers) => { + const status = headers[":status"] as number; + req.on("data", () => {}); + req.on("end", () => resolve({ latencyMs: performance.now() - start, status })); + }); + req.on("error", reject); + req.end(body); + }); +} + +async function main() { + const url = new URL(BASE_URL); + const client = http2.connect(url.origin); + await new Promise((resolve, reject) => { + client.on("connect", resolve); + client.on("error", reject); + }); + + const maxStreams = client.remoteSettings?.maxConcurrentStreams; + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams}`); + + // Warmup + const w = await sendRequest(client); + console.log(`Warmup: status=${w.status}, latency=${w.latencyMs.toFixed(0)}ms`); + + // Burst 50 requests on single warmed connection + const count = 50; + console.log(`\nBursting ${count} requests on 1 warmed connection...`); + const wallStart = performance.now(); + const results = await Promise.all(Array.from({ length: count }, () => sendRequest(client))); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + client.close(); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/loadtest/h2-test.ts b/loadtest/h2-test.ts new file mode 100644 index 000000000..7b31b5db4 --- /dev/null +++ b/loadtest/h2-test.ts @@ -0,0 +1,112 @@ +import http2 from "node:http2"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "10", 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-h2-0", + environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, + metadata: { test_run: "h2", index: "0" }, + launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, +}); + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +function connectH2(origin: string): Promise { + return new Promise((resolve, reject) => { + const client = http2.connect(origin); + client.on("connect", () => resolve(client)); + client.on("error", reject); + }); +} + +function sendRequest( + client: http2.ClientHttp2Session, +): Promise<{ latencyMs: number; status: number }> { + return new Promise((resolve, reject) => { + const start = performance.now(); + const req = client.request({ + ":method": "POST", + ":path": "/v1/devboxes", + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }); + req.on("response", (headers) => { + const status = headers[":status"] as number; + req.on("data", () => {}); + req.on("end", () => resolve({ latencyMs: performance.now() - start, status })); + }); + req.on("error", reject); + req.end(body); + }); +} + +async function main() { + console.log( + `HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`, + ); + + const url = new URL(BASE_URL); + const clients = await Promise.all( + Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin)), + ); + + const maxStreams = clients[0].remoteSettings?.maxConcurrentStreams; + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? "unknown"}`); + console.log(`${NUM_CONNECTIONS} connections established\n`); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, 2000); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => { + const client = clients[i % NUM_CONNECTIONS]; + return sendRequest(client).then((r) => { + completed++; + return r; + }); + }); + + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + clearInterval(progressTimer); + + for (const c of clients) c.close(); + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\n=== HTTP/2 Results ===`); + console.log(`Requests: ${REQUEST_COUNT}`); + console.log(`Connections: ${NUM_CONNECTIONS}`); + console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/loadtest.ts b/loadtest/loadtest.ts new file mode 100644 index 000000000..1c42e970d --- /dev/null +++ b/loadtest/loadtest.ts @@ -0,0 +1,159 @@ +import { Runloop } from "@runloop/api-client"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "100000", 10); +const RUNLOOP_BASE_URL = process.env.RUNLOOP_BASE_URL; +const USE_HTTP2 = process.env.USE_HTTP2 === "1"; +const PROGRESS_INTERVAL_MS = 2000; + +function buildClient(): Runloop { + const opts: ConstructorParameters[0] = { + maxRetries: 0, + timeout: 120_000, + }; + if (RUNLOOP_BASE_URL) { + opts.baseURL = RUNLOOP_BASE_URL; + } + if (USE_HTTP2) { + (opts as any).http2 = true; + } + return new Runloop(opts); +} + +interface RequestResult { + index: number; + latencyMs: number; + status: number | null; + error: string | null; +} + +async function sendRequest( + client: Runloop, + index: number, + runId: string, +): Promise { + const start = performance.now(); + try { + await client.devboxes.create({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: `loadtest-${runId}-${index}`, + environment_variables: { + TEST_VAR_1: "value_one", + TEST_VAR_2: "value_two", + }, + metadata: { + test_run: runId, + index: String(index), + }, + launch_parameters: { + resource_size_request: "SMALL", + keep_alive_time_seconds: 300, + }, + }); + return { + index, + latencyMs: performance.now() - start, + status: 200, + error: null, + }; + } catch (err: any) { + return { + index, + latencyMs: performance.now() - start, + status: err?.status ?? null, + error: err?.message ?? String(err), + }; + } +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +function printMetrics(results: RequestResult[], wallClockMs: number): void { + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + + const statusCounts = new Map(); + for (const r of results) { + const key = r.status != null ? String(r.status) : "network_error"; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); + } + + console.log("\n=== Load Test Results ==="); + console.log(`Requests: ${results.length}`); + console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`); + console.log( + `Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`, + ); + console.log(""); + console.log("Latency (ms):"); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(""); + console.log("Status codes:"); + for (const [status, count] of [...statusCounts.entries()].sort()) { + console.log(` ${status}: ${count}`); + } +} + +async function main(): Promise { + const fdLimit = await checkFileDescriptorLimit(); + if (!USE_HTTP2 && fdLimit < 10000) { + console.warn( + `\nWARNING: File descriptor limit is ${fdLimit}. For 100k HTTP/1.1 requests, run:\n` + + ` ulimit -n 65536\n` + + `Or use HTTP/2 multiplexing: USE_HTTP2=1\n`, + ); + } + + const client = buildClient(); + const runId = `run-${Date.now()}`; + + console.log(`Starting load test: ${REQUEST_COUNT} concurrent requests`); + console.log(`Run ID: ${runId}`); + console.log(`HTTP mode: ${USE_HTTP2 ? "HTTP/2 (undici)" : "HTTP/1.1 (node-fetch)"}`); + console.log(`Base URL: ${RUNLOOP_BASE_URL ?? "(SDK default)"}`); + console.log(`File descriptor limit: ${fdLimit}`); + console.log(""); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, PROGRESS_INTERVAL_MS); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => + sendRequest(client, i, runId).then((result) => { + completed++; + return result; + }), + ); + + const results = await Promise.all(promises); + + const wallClockMs = performance.now() - wallStart; + clearInterval(progressTimer); + + printMetrics(results, wallClockMs); +} + +async function checkFileDescriptorLimit(): Promise { + try { + const { execSync } = await import("child_process"); + return parseInt(execSync("ulimit -n", { encoding: "utf-8" }).trim(), 10); + } catch { + return -1; + } +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/loadtest/package.json b/loadtest/package.json new file mode 100644 index 000000000..91f0a86bd --- /dev/null +++ b/loadtest/package.json @@ -0,0 +1,16 @@ +{ + "name": "runloop-loadtest", + "private": true, + "type": "module", + "scripts": { + "test": "tsx loadtest.ts" + }, + "dependencies": { + "@runloop/api-client": "file:..", + "tsx": "^4.0.0", + "undici": "^7.26.0" + }, + "devDependencies": { + "@types/node": "^25.9.3" + } +} diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts new file mode 100644 index 000000000..ebcaf2a93 --- /dev/null +++ b/loadtest/raw-fetch-test.ts @@ -0,0 +1,84 @@ +import https from "node:https"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const agent = new https.Agent({ + keepAlive: true, + maxSockets: Infinity, + maxFreeSockets: 4096, +}); + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-raw-0", + environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, + metadata: { test_run: "raw", index: "0" }, + launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, +}); + +function makeRequest(index: number): Promise<{ latencyMs: number; status: number }> { + const start = performance.now(); + return new Promise((resolve, reject) => { + const url = new URL("/v1/devboxes", BASE_URL); + const req = https.request( + url, + { + method: "POST", + agent, + headers: { + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + "content-length": Buffer.byteLength(body).toString(), + }, + }, + (res) => { + res.resume(); + res.on("end", () => + resolve({ latencyMs: performance.now() - start, status: res.statusCode! }), + ); + }, + ); + req.on("error", reject); + req.end(body); + }); +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +async function main() { + console.log(`Raw node:https test: ${REQUEST_COUNT} concurrent requests to ${BASE_URL}`); + + const wallStart = performance.now(); + const results = await Promise.all( + Array.from({ length: REQUEST_COUNT }, (_, i) => makeRequest(i)), + ); + const wallMs = performance.now() - wallStart; + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\nWall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-debug.ts b/loadtest/undici-debug.ts new file mode 100644 index 000000000..ab5286855 --- /dev/null +++ b/loadtest/undici-debug.ts @@ -0,0 +1,72 @@ +import { Agent, fetch } from "undici"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-debug-0", + environment_variables: { TEST_VAR_1: "value_one" }, + launch_parameters: { resource_size_request: "SMALL" }, +}); + +async function main() { + console.log("=== undici HTTP/2 debug ===\n"); + + const dispatcher = new Agent({ + allowH2: true, + connections: 2, + pipelining: 10, + keepAliveTimeout: 600_000, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Single request to inspect protocol + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + + console.log("Status:", res.status); + console.log("HTTP version:", res.headers.get("x-http-version") ?? "(not reported)"); + console.log("\nResponse headers:"); + for (const [k, v] of res.headers) { + console.log(` ${k}: ${v}`); + } + await res.text(); + + // Quick 20-request burst to verify concurrency + console.log("\n--- 20-request burst (2 connections, pipelining=10) ---"); + const wallStart = performance.now(); + const promises = Array.from({ length: 20 }, async () => { + const start = performance.now(); + const r = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await r.text(); + return { latencyMs: performance.now() - start, status: r.status }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + console.log(`Wall clock: ${wallMs.toFixed(0)}ms`); + console.log(`Throughput: ${(20 / (wallMs / 1000)).toFixed(1)} req/s`); + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`Latency: min=${lats[0].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await dispatcher.close(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-single-conn.ts b/loadtest/undici-single-conn.ts new file mode 100644 index 000000000..434e7a1d7 --- /dev/null +++ b/loadtest/undici-single-conn.ts @@ -0,0 +1,107 @@ +import { Client, Agent, fetch } from "undici"; + +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; +const PIPELINING = parseInt(process.env.PIPELINING ?? "100", 10); + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-single-0", + environment_variables: { TEST_VAR_1: "value_one" }, + launch_parameters: { resource_size_request: "SMALL" }, +}); + +async function testClient() { + console.log(`\n--- undici.Client (single connection, pipelining=${PIPELINING}) ---`); + const url = new URL(BASE_URL); + const client = new Client(url.origin, { + allowH2: true, + pipelining: PIPELINING, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Warm up - single request to establish connection + const warmup = await client.request({ + method: "POST", + path: "/v1/devboxes", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + }); + await warmup.body.text(); + console.log(`Warmup: status=${warmup.statusCode}`); + + // Burst 50 requests on single connection + const count = 50; + const wallStart = performance.now(); + const promises = Array.from({ length: count }, async () => { + const start = performance.now(); + const res = await client.request({ + method: "POST", + path: "/v1/devboxes", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + }); + await res.body.text(); + return { latencyMs: performance.now() - start, status: res.statusCode }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await client.close(); +} + +async function testAgent() { + console.log(`\n--- undici.Agent (connections=1, pipelining=${PIPELINING}) ---`); + const dispatcher = new Agent({ + allowH2: true, + connections: 1, + pipelining: PIPELINING, + bodyTimeout: 0, + headersTimeout: 0, + }); + + // Warm up + const warmup = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await warmup.text(); + console.log(`Warmup: status=${warmup.status}`); + + // Burst 50 requests + const count = 50; + const wallStart = performance.now(); + const promises = Array.from({ length: count }, async () => { + const start = performance.now(); + const r = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + body, + dispatcher, + } as any); + await r.text(); + return { latencyMs: performance.now() - start, status: r.status }; + }); + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + + const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); + console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); + console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + + await dispatcher.close(); +} + +async function main() { + await testClient(); + await testAgent(); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/loadtest/undici-test.ts b/loadtest/undici-test.ts new file mode 100644 index 000000000..0a616e198 --- /dev/null +++ b/loadtest/undici-test.ts @@ -0,0 +1,93 @@ +import { Agent, fetch } from "undici"; + +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "20", 10); +const PIPELINING = parseInt(process.env.PIPELINING ?? "128", 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const API_KEY = process.env.RUNLOOP_API_KEY!; + +const body = JSON.stringify({ + blueprint_id: "bp_nonexistent_loadtest_00000", + name: "loadtest-undici-0", + environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, + metadata: { test_run: "undici", index: "0" }, + launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, +}); + +function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +async function main() { + const dispatcher = new Agent({ + allowH2: true, + connections: NUM_CONNECTIONS, + pipelining: PIPELINING, + keepAliveTimeout: 600_000, + keepAliveMaxTimeout: 600_000, + bodyTimeout: 0, + headersTimeout: 0, + }); + + console.log( + `undici direct test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections, pipelining=${PIPELINING}`, + ); + console.log(`Target: ${BASE_URL}`); + + let completed = 0; + const progressTimer = setInterval(() => { + console.log( + ` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`, + ); + }, 2000); + + const wallStart = performance.now(); + + const promises = Array.from({ length: REQUEST_COUNT }, async () => { + const start = performance.now(); + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + await res.text(); + completed++; + return { latencyMs: performance.now() - start, status: res.status }; + }); + + const results = await Promise.all(promises); + const wallMs = performance.now() - wallStart; + clearInterval(progressTimer); + + await dispatcher.close(); + + const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); + const statusCounts = new Map(); + for (const r of results) { + statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + } + + console.log(`\n=== undici Direct Results ===`); + console.log(`Requests: ${REQUEST_COUNT}`); + console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); + console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(`\nStatus codes:`); + for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); From fb0467767e3e71c309dc5b0128f12a88137c75f2 Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Thu, 11 Jun 2026 17:00:42 -0700 Subject: [PATCH 2/5] fix: handle response-level errors in raw-fetch loadtest Add res.on('error', reject) so the promise settles if the server aborts mid-response, instead of hanging indefinitely. Co-Authored-By: Claude Opus 4.6 --- loadtest/raw-fetch-test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts index ebcaf2a93..4a2e78adf 100644 --- a/loadtest/raw-fetch-test.ts +++ b/loadtest/raw-fetch-test.ts @@ -38,6 +38,7 @@ function makeRequest(index: number): Promise<{ latencyMs: number; status: number res.on("end", () => resolve({ latencyMs: performance.now() - start, status: res.statusCode! }), ); + res.on("error", reject); }, ); req.on("error", reject); From 8602fbd4a92fb388ae91c9f5b73326c8d6bab241 Mon Sep 17 00:00:00 2001 From: Reflex Date: Fri, 10 Jul 2026 12:07:34 +0000 Subject: [PATCH 3/5] test(loadtest): run against in-repo SDK and match HTTP/2-as-default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the SDK load harness for the new default transport: - `loadtest.ts` imports `../src/sdk.ts` (like the sibling loadtest scripts) so it always exercises the SDK code in this checkout, not a published build. - Set `http2` explicitly: HTTP/2 is now the default, so HTTP/1.1 must be opted into with `http2: false`. `USE_HTTP2` defaults to on; `USE_HTTP2=0` benchmarks HTTP/1.1. Relabel the HTTP/2 mode as native `node:http2`. - Drop the `@runloop/api-client` file dependency and `type: module` from `loadtest/package.json` (the latter would force the shared loadtest dir to ESM and break `push-to-loki.ts`); keep `undici` so the raw undici-vs-native comparison still runs — that comparison motivated the native H2 transport. - Document the real-API comparison scripts (SDK + raw undici/node:http2/https) in the loadtest README. Co-Authored-By: Claude Opus 4.8 --- loadtest/README.md | 50 ++++++++++++++++++++++++++++++------- loadtest/loadtest.ts | 57 ++++++++++++++++++++----------------------- loadtest/package.json | 2 -- 3 files changed, 68 insertions(+), 41 deletions(-) diff --git a/loadtest/README.md b/loadtest/README.md index 7ab30bce0..af65ec61b 100644 --- a/loadtest/README.md +++ b/loadtest/README.md @@ -6,15 +6,47 @@ duration. Not part of `npm test`; run on demand. ## Scripts -| Script | Purpose | -|---|---| -| `h2-multiplex.ts` | Fire N concurrent requests, report throughput + p50/p95/p99. | -| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time. | -| `h2-leak.ts` | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection. | -| `h2-chaos.ts` | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. | -| `h2load.sh` | Wraps `nghttp2`'s `h2load` against a long-running test server. | -| `sse-test.ts` | Pre-existing manual SSE round-trip. | -| `push-to-loki.ts` | Runs all four timed tests and pushes results to Loki for Grafana. | +| Script | Purpose | +| ------------------- | ----------------------------------------------------------------------------------- | +| `h2-multiplex.ts` | Fire N concurrent requests, report throughput + p50/p95/p99. | +| `h2-pool-growth.ts` | Ramp 1 → 50 → 200 → 50 → 1 concurrent. Observe pool size over time. | +| `h2-leak.ts` | Soak at 50 r/s. Sample heap + FD count. Periodic GOAWAY injection. | +| `h2-chaos.ts` | Server randomly drops sockets / RSTs / GOAWAYs / delays. Asserts ≥50 % GET success. | +| `h2load.sh` | Wraps `nghttp2`'s `h2load` against a long-running test server. | +| `sse-test.ts` | Pre-existing manual SSE round-trip. | +| `push-to-loki.ts` | Runs all four timed tests and pushes results to Loki for Grafana. | + +## End-to-end transport comparison (real API) + +Unlike the in-process scripts above, these hit a real Runloop API (`RUNLOOP_API_KEY`, +optional `RUNLOOP_BASE_URL`). They send a burst of `devboxes.create` calls against a +**deliberately nonexistent blueprint** (`bp_nonexistent_loadtest_00000`), so every +request fails fast server-side (HTTP `400`) and **no devboxes are created** — isolating +client + server _request handling_ from provisioning. + +| Script | Transport under test | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `loadtest.ts` | The **SDK** itself. `USE_HTTP2=0` → HTTP/1.1 (`node-fetch`); default / `USE_HTTP2=1` → HTTP/2 (native `node:http2`). Imports `../src/sdk.ts`, so it always runs against the SDK code in this checkout. | +| `undici-test.ts`, `undici-single-conn.ts`, `undici-debug.ts` | Raw [undici](https://github.com/nodejs/undici) HTTP/2 pool, bypassing the SDK. | +| `h2-test.ts`, `h2-single-conn.ts` | Raw `node:http2`, bypassing the SDK. | +| `raw-fetch-test.ts` | Raw `node:https` (HTTP/1.1 keep-alive agent). | +| `alpn-check.ts` | Confirms the origin negotiates `h2` via TLS ALPN. | + +The raw-transport probes compare undici's HTTP/2 multiplexing against `node:http2` +(and HTTP/1.1) directly — the comparison that motivated building the SDK's own +`node:http2` transport. They're kept so that comparison stays reproducible. + +```sh +# SDK: HTTP/2 (default) vs HTTP/1.1, 2000-request burst +source ~/env && REQUEST_COUNT=2000 npx tsx loadtest/loadtest.ts # HTTP/2 +source ~/env && REQUEST_COUNT=2000 USE_HTTP2=0 npx tsx loadtest/loadtest.ts # HTTP/1.1 + +# Raw undici vs node:http2 comparison (undici comes from loadtest/package.json) +cd loadtest && npm install && source ~/env && npx tsx undici-test.ts +``` + +HTTP/1.1 opens a socket per in-flight request; for large bursts raise the file-descriptor +limit (`ulimit -n 65536`) or keep `REQUEST_COUNT` under it. ## Automated daily runs diff --git a/loadtest/loadtest.ts b/loadtest/loadtest.ts index 1c42e970d..e75775885 100644 --- a/loadtest/loadtest.ts +++ b/loadtest/loadtest.ts @@ -1,8 +1,11 @@ -import { Runloop } from "@runloop/api-client"; +// Import the in-repo SDK source directly (run via `npx tsx`) so the benchmark +// always exercises the exact code in this checkout — not a published build. +import { Runloop } from '../src/sdk.ts'; -const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "100000", 10); +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '100000', 10); const RUNLOOP_BASE_URL = process.env.RUNLOOP_BASE_URL; -const USE_HTTP2 = process.env.USE_HTTP2 === "1"; +// HTTP/2 is the SDK's default transport; set USE_HTTP2=0 to benchmark HTTP/1.1. +const USE_HTTP2 = (process.env.USE_HTTP2 ?? '1') === '1'; const PROGRESS_INTERVAL_MS = 2000; function buildClient(): Runloop { @@ -13,9 +16,9 @@ function buildClient(): Runloop { if (RUNLOOP_BASE_URL) { opts.baseURL = RUNLOOP_BASE_URL; } - if (USE_HTTP2) { - (opts as any).http2 = true; - } + // Set explicitly: since HTTP/2 is the default, HTTP/1.1 must be opted into + // with `http2: false` (omitting it would now select HTTP/2). + (opts as any).http2 = USE_HTTP2; return new Runloop(opts); } @@ -26,26 +29,22 @@ interface RequestResult { error: string | null; } -async function sendRequest( - client: Runloop, - index: number, - runId: string, -): Promise { +async function sendRequest(client: Runloop, index: number, runId: string): Promise { const start = performance.now(); try { await client.devboxes.create({ - blueprint_id: "bp_nonexistent_loadtest_00000", + blueprint_id: 'bp_nonexistent_loadtest_00000', name: `loadtest-${runId}-${index}`, environment_variables: { - TEST_VAR_1: "value_one", - TEST_VAR_2: "value_two", + TEST_VAR_1: 'value_one', + TEST_VAR_2: 'value_two', }, metadata: { test_run: runId, index: String(index), }, launch_parameters: { - resource_size_request: "SMALL", + resource_size_request: 'SMALL', keep_alive_time_seconds: 300, }, }); @@ -75,26 +74,24 @@ function printMetrics(results: RequestResult[], wallClockMs: number): void { const statusCounts = new Map(); for (const r of results) { - const key = r.status != null ? String(r.status) : "network_error"; + const key = r.status != null ? String(r.status) : 'network_error'; statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); } - console.log("\n=== Load Test Results ==="); + console.log('\n=== Load Test Results ==='); console.log(`Requests: ${results.length}`); console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`); - console.log( - `Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`, - ); - console.log(""); - console.log("Latency (ms):"); + console.log(`Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`); + console.log(''); + console.log('Latency (ms):'); console.log(` min: ${latencies[0].toFixed(1)}`); console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); - console.log(""); - console.log("Status codes:"); + console.log(''); + console.log('Status codes:'); for (const [status, count] of [...statusCounts.entries()].sort()) { console.log(` ${status}: ${count}`); } @@ -115,10 +112,10 @@ async function main(): Promise { console.log(`Starting load test: ${REQUEST_COUNT} concurrent requests`); console.log(`Run ID: ${runId}`); - console.log(`HTTP mode: ${USE_HTTP2 ? "HTTP/2 (undici)" : "HTTP/1.1 (node-fetch)"}`); - console.log(`Base URL: ${RUNLOOP_BASE_URL ?? "(SDK default)"}`); + console.log(`HTTP mode: ${USE_HTTP2 ? 'HTTP/2 (SDK native node:http2)' : 'HTTP/1.1 (node-fetch)'}`); + console.log(`Base URL: ${RUNLOOP_BASE_URL ?? '(SDK default)'}`); console.log(`File descriptor limit: ${fdLimit}`); - console.log(""); + console.log(''); let completed = 0; const progressTimer = setInterval(() => { @@ -146,14 +143,14 @@ async function main(): Promise { async function checkFileDescriptorLimit(): Promise { try { - const { execSync } = await import("child_process"); - return parseInt(execSync("ulimit -n", { encoding: "utf-8" }).trim(), 10); + const { execSync } = await import('child_process'); + return parseInt(execSync('ulimit -n', { encoding: 'utf-8' }).trim(), 10); } catch { return -1; } } main().catch((err) => { - console.error("Fatal error:", err); + console.error('Fatal error:', err); process.exit(1); }); diff --git a/loadtest/package.json b/loadtest/package.json index 91f0a86bd..aa8b2d361 100644 --- a/loadtest/package.json +++ b/loadtest/package.json @@ -1,12 +1,10 @@ { "name": "runloop-loadtest", "private": true, - "type": "module", "scripts": { "test": "tsx loadtest.ts" }, "dependencies": { - "@runloop/api-client": "file:..", "tsx": "^4.0.0", "undici": "^7.26.0" }, From 66fe8dc4c0f3fa7bf9eea7dc7aae1045517f234f Mon Sep 17 00:00:00 2001 From: Reflex Date: Fri, 10 Jul 2026 12:10:35 +0000 Subject: [PATCH 4/5] style(loadtest): prettier-format raw-transport comparison scripts The raw undici / node:http2 / node:https probes predated the repo's prettier/eslint style (double quotes), which fails `eslint .`. Format them so the rebased branch is lint-clean; no behavior change. Co-Authored-By: Claude Opus 4.8 --- loadtest/alpn-check.ts | 12 ++++---- loadtest/h2-single-conn.ts | 41 +++++++++++++++------------ loadtest/h2-test.ts | 52 +++++++++++++++------------------- loadtest/raw-fetch-test.ts | 36 +++++++++++------------ loadtest/undici-debug.ts | 30 ++++++++++---------- loadtest/undici-single-conn.ts | 47 +++++++++++++++++------------- loadtest/undici-test.ts | 24 ++++++++-------- 7 files changed, 122 insertions(+), 120 deletions(-) diff --git a/loadtest/alpn-check.ts b/loadtest/alpn-check.ts index 9a59813c1..678c83573 100644 --- a/loadtest/alpn-check.ts +++ b/loadtest/alpn-check.ts @@ -1,6 +1,6 @@ -import tls from "node:tls"; +import tls from 'node:tls'; -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const url = new URL(BASE_URL); console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`); @@ -8,8 +8,8 @@ console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`); const socket = tls.connect( { host: url.hostname, - port: parseInt(url.port || "443", 10), - ALPNProtocols: ["h2", "http/1.1"], + port: parseInt(url.port || '443', 10), + ALPNProtocols: ['h2', 'http/1.1'], servername: url.hostname, }, () => { @@ -19,6 +19,6 @@ const socket = tls.connect( }, ); -socket.on("error", (err) => { - console.error("TLS error:", err.message); +socket.on('error', (err) => { + console.error('TLS error:', err.message); }); diff --git a/loadtest/h2-single-conn.ts b/loadtest/h2-single-conn.ts index 7ceb78ed1..e563274e2 100644 --- a/loadtest/h2-single-conn.ts +++ b/loadtest/h2-single-conn.ts @@ -1,30 +1,30 @@ -import http2 from "node:http2"; +import http2 from 'node:http2'; -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const API_KEY = process.env.RUNLOOP_API_KEY!; const body = JSON.stringify({ - blueprint_id: "bp_nonexistent_loadtest_00000", - name: "loadtest-h2s-0", - environment_variables: { TEST_VAR_1: "value_one" }, - launch_parameters: { resource_size_request: "SMALL" }, + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-h2s-0', + environment_variables: { TEST_VAR_1: 'value_one' }, + launch_parameters: { resource_size_request: 'SMALL' }, }); function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> { return new Promise((resolve, reject) => { const start = performance.now(); const req = client.request({ - ":method": "POST", - ":path": "/v1/devboxes", - "content-type": "application/json", + ':method': 'POST', + ':path': '/v1/devboxes', + 'content-type': 'application/json', authorization: `Bearer ${API_KEY}`, }); - req.on("response", (headers) => { - const status = headers[":status"] as number; - req.on("data", () => {}); - req.on("end", () => resolve({ latencyMs: performance.now() - start, status })); + req.on('response', (headers) => { + const status = headers[':status'] as number; + req.on('data', () => {}); + req.on('end', () => resolve({ latencyMs: performance.now() - start, status })); }); - req.on("error", reject); + req.on('error', reject); req.end(body); }); } @@ -33,8 +33,8 @@ async function main() { const url = new URL(BASE_URL); const client = http2.connect(url.origin); await new Promise((resolve, reject) => { - client.on("connect", resolve); - client.on("error", reject); + client.on('connect', resolve); + client.on('error', reject); }); const maxStreams = client.remoteSettings?.maxConcurrentStreams; @@ -53,9 +53,14 @@ async function main() { const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); - console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + console.log( + `Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`, + ); client.close(); } -main().catch((e) => { console.error(e); process.exit(1); }); +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/h2-test.ts b/loadtest/h2-test.ts index 7b31b5db4..7922e3fcd 100644 --- a/loadtest/h2-test.ts +++ b/loadtest/h2-test.ts @@ -1,16 +1,16 @@ -import http2 from "node:http2"; +import http2 from 'node:http2'; -const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); -const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "10", 10); -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? '10', 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const API_KEY = process.env.RUNLOOP_API_KEY!; const body = JSON.stringify({ - blueprint_id: "bp_nonexistent_loadtest_00000", - name: "loadtest-h2-0", - environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, - metadata: { test_run: "h2", index: "0" }, - launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-h2-0', + environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' }, + metadata: { test_run: 'h2', index: '0' }, + launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, }); function percentile(sorted: number[], p: number): number { @@ -21,44 +21,38 @@ function percentile(sorted: number[], p: number): number { function connectH2(origin: string): Promise { return new Promise((resolve, reject) => { const client = http2.connect(origin); - client.on("connect", () => resolve(client)); - client.on("error", reject); + client.on('connect', () => resolve(client)); + client.on('error', reject); }); } -function sendRequest( - client: http2.ClientHttp2Session, -): Promise<{ latencyMs: number; status: number }> { +function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> { return new Promise((resolve, reject) => { const start = performance.now(); const req = client.request({ - ":method": "POST", - ":path": "/v1/devboxes", - "content-type": "application/json", + ':method': 'POST', + ':path': '/v1/devboxes', + 'content-type': 'application/json', authorization: `Bearer ${API_KEY}`, }); - req.on("response", (headers) => { - const status = headers[":status"] as number; - req.on("data", () => {}); - req.on("end", () => resolve({ latencyMs: performance.now() - start, status })); + req.on('response', (headers) => { + const status = headers[':status'] as number; + req.on('data', () => {}); + req.on('end', () => resolve({ latencyMs: performance.now() - start, status })); }); - req.on("error", reject); + req.on('error', reject); req.end(body); }); } async function main() { - console.log( - `HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`, - ); + console.log(`HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`); const url = new URL(BASE_URL); - const clients = await Promise.all( - Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin)), - ); + const clients = await Promise.all(Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin))); const maxStreams = clients[0].remoteSettings?.maxConcurrentStreams; - console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? "unknown"}`); + console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? 'unknown'}`); console.log(`${NUM_CONNECTIONS} connections established\n`); let completed = 0; diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts index 4a2e78adf..c55bb80a9 100644 --- a/loadtest/raw-fetch-test.ts +++ b/loadtest/raw-fetch-test.ts @@ -1,7 +1,7 @@ -import https from "node:https"; +import https from 'node:https'; -const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const API_KEY = process.env.RUNLOOP_API_KEY!; const agent = new https.Agent({ @@ -11,37 +11,35 @@ const agent = new https.Agent({ }); const body = JSON.stringify({ - blueprint_id: "bp_nonexistent_loadtest_00000", - name: "loadtest-raw-0", - environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, - metadata: { test_run: "raw", index: "0" }, - launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-raw-0', + environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' }, + metadata: { test_run: 'raw', index: '0' }, + launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, }); function makeRequest(index: number): Promise<{ latencyMs: number; status: number }> { const start = performance.now(); return new Promise((resolve, reject) => { - const url = new URL("/v1/devboxes", BASE_URL); + const url = new URL('/v1/devboxes', BASE_URL); const req = https.request( url, { - method: "POST", + method: 'POST', agent, headers: { - "content-type": "application/json", + 'content-type': 'application/json', authorization: `Bearer ${API_KEY}`, - "content-length": Buffer.byteLength(body).toString(), + 'content-length': Buffer.byteLength(body).toString(), }, }, (res) => { res.resume(); - res.on("end", () => - resolve({ latencyMs: performance.now() - start, status: res.statusCode! }), - ); - res.on("error", reject); + res.on('end', () => resolve({ latencyMs: performance.now() - start, status: res.statusCode! })); + res.on('error', reject); }, ); - req.on("error", reject); + req.on('error', reject); req.end(body); }); } @@ -55,9 +53,7 @@ async function main() { console.log(`Raw node:https test: ${REQUEST_COUNT} concurrent requests to ${BASE_URL}`); const wallStart = performance.now(); - const results = await Promise.all( - Array.from({ length: REQUEST_COUNT }, (_, i) => makeRequest(i)), - ); + const results = await Promise.all(Array.from({ length: REQUEST_COUNT }, (_, i) => makeRequest(i))); const wallMs = performance.now() - wallStart; const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); diff --git a/loadtest/undici-debug.ts b/loadtest/undici-debug.ts index ab5286855..17c387ebc 100644 --- a/loadtest/undici-debug.ts +++ b/loadtest/undici-debug.ts @@ -1,17 +1,17 @@ -import { Agent, fetch } from "undici"; +import { Agent, fetch } from 'undici'; -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const API_KEY = process.env.RUNLOOP_API_KEY!; const body = JSON.stringify({ - blueprint_id: "bp_nonexistent_loadtest_00000", - name: "loadtest-debug-0", - environment_variables: { TEST_VAR_1: "value_one" }, - launch_parameters: { resource_size_request: "SMALL" }, + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-debug-0', + environment_variables: { TEST_VAR_1: 'value_one' }, + launch_parameters: { resource_size_request: 'SMALL' }, }); async function main() { - console.log("=== undici HTTP/2 debug ===\n"); + console.log('=== undici HTTP/2 debug ===\n'); const dispatcher = new Agent({ allowH2: true, @@ -24,31 +24,31 @@ async function main() { // Single request to inspect protocol const res = await fetch(`${BASE_URL}/v1/devboxes`, { - method: "POST", + method: 'POST', headers: { - "content-type": "application/json", + 'content-type': 'application/json', authorization: `Bearer ${API_KEY}`, }, body, dispatcher, } as any); - console.log("Status:", res.status); - console.log("HTTP version:", res.headers.get("x-http-version") ?? "(not reported)"); - console.log("\nResponse headers:"); + console.log('Status:', res.status); + console.log('HTTP version:', res.headers.get('x-http-version') ?? '(not reported)'); + console.log('\nResponse headers:'); for (const [k, v] of res.headers) { console.log(` ${k}: ${v}`); } await res.text(); // Quick 20-request burst to verify concurrency - console.log("\n--- 20-request burst (2 connections, pipelining=10) ---"); + console.log('\n--- 20-request burst (2 connections, pipelining=10) ---'); const wallStart = performance.now(); const promises = Array.from({ length: 20 }, async () => { const start = performance.now(); const r = await fetch(`${BASE_URL}/v1/devboxes`, { - method: "POST", - headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, body, dispatcher, } as any); diff --git a/loadtest/undici-single-conn.ts b/loadtest/undici-single-conn.ts index 434e7a1d7..b85cc0fd5 100644 --- a/loadtest/undici-single-conn.ts +++ b/loadtest/undici-single-conn.ts @@ -1,14 +1,14 @@ -import { Client, Agent, fetch } from "undici"; +import { Client, Agent, fetch } from 'undici'; -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const API_KEY = process.env.RUNLOOP_API_KEY!; -const PIPELINING = parseInt(process.env.PIPELINING ?? "100", 10); +const PIPELINING = parseInt(process.env.PIPELINING ?? '100', 10); const body = JSON.stringify({ - blueprint_id: "bp_nonexistent_loadtest_00000", - name: "loadtest-single-0", - environment_variables: { TEST_VAR_1: "value_one" }, - launch_parameters: { resource_size_request: "SMALL" }, + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-single-0', + environment_variables: { TEST_VAR_1: 'value_one' }, + launch_parameters: { resource_size_request: 'SMALL' }, }); async function testClient() { @@ -23,9 +23,9 @@ async function testClient() { // Warm up - single request to establish connection const warmup = await client.request({ - method: "POST", - path: "/v1/devboxes", - headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + method: 'POST', + path: '/v1/devboxes', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, body, }); await warmup.body.text(); @@ -37,9 +37,9 @@ async function testClient() { const promises = Array.from({ length: count }, async () => { const start = performance.now(); const res = await client.request({ - method: "POST", - path: "/v1/devboxes", - headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + method: 'POST', + path: '/v1/devboxes', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, body, }); await res.body.text(); @@ -50,7 +50,9 @@ async function testClient() { const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); - console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + console.log( + `Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`, + ); await client.close(); } @@ -67,8 +69,8 @@ async function testAgent() { // Warm up const warmup = await fetch(`${BASE_URL}/v1/devboxes`, { - method: "POST", - headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, body, dispatcher, } as any); @@ -81,8 +83,8 @@ async function testAgent() { const promises = Array.from({ length: count }, async () => { const start = performance.now(); const r = await fetch(`${BASE_URL}/v1/devboxes`, { - method: "POST", - headers: { "content-type": "application/json", authorization: `Bearer ${API_KEY}` }, + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${API_KEY}` }, body, dispatcher, } as any); @@ -94,7 +96,9 @@ async function testAgent() { const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b); console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`); - console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`); + console.log( + `Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`, + ); await dispatcher.close(); } @@ -104,4 +108,7 @@ async function main() { await testAgent(); } -main().catch((e) => { console.error(e); process.exit(1); }); +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/loadtest/undici-test.ts b/loadtest/undici-test.ts index 0a616e198..f0c42292e 100644 --- a/loadtest/undici-test.ts +++ b/loadtest/undici-test.ts @@ -1,17 +1,17 @@ -import { Agent, fetch } from "undici"; +import { Agent, fetch } from 'undici'; -const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10); -const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "20", 10); -const PIPELINING = parseInt(process.env.PIPELINING ?? "128", 10); -const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro"; +const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? '10000', 10); +const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? '20', 10); +const PIPELINING = parseInt(process.env.PIPELINING ?? '128', 10); +const BASE_URL = process.env.RUNLOOP_BASE_URL ?? 'https://api.runloop.pro'; const API_KEY = process.env.RUNLOOP_API_KEY!; const body = JSON.stringify({ - blueprint_id: "bp_nonexistent_loadtest_00000", - name: "loadtest-undici-0", - environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" }, - metadata: { test_run: "undici", index: "0" }, - launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 }, + blueprint_id: 'bp_nonexistent_loadtest_00000', + name: 'loadtest-undici-0', + environment_variables: { TEST_VAR_1: 'value_one', TEST_VAR_2: 'value_two' }, + metadata: { test_run: 'undici', index: '0' }, + launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, }); function percentile(sorted: number[], p: number): number { @@ -47,9 +47,9 @@ async function main() { const promises = Array.from({ length: REQUEST_COUNT }, async () => { const start = performance.now(); const res = await fetch(`${BASE_URL}/v1/devboxes`, { - method: "POST", + method: 'POST', headers: { - "content-type": "application/json", + 'content-type': 'application/json', authorization: `Bearer ${API_KEY}`, }, body, From f2cd19a8d796f7e61ac4cd4cc428c7235871e175 Mon Sep 17 00:00:00 2001 From: Reflex Date: Fri, 10 Jul 2026 12:59:45 +0000 Subject: [PATCH 5/5] fix(loadtest): make comparison scripts robust to failures and empty input Address CodeAnt review on the loadtest harness: - undici-test / raw-fetch-test: collect per-request results instead of `Promise.all` fail-fast. One transient network error (or a mid-stream response abort in raw-fetch) no longer hangs or discards the whole run; failures are counted as `network_error` in the status breakdown. - raw-fetch-test: settle each request exactly once across end / aborted / error / request-error paths (previously a mid-stream abort could hang the batch indefinitely). - h2-test: reject NUM_CONNECTIONS < 1 up front (previously dereferenced clients[0] === undefined and crashed). - All scripts: guard the latency block against zero results so an empty run doesn't throw on latencies[0]. Co-Authored-By: Claude Opus 4.8 --- loadtest/h2-test.ts | 21 ++++++++++----- loadtest/loadtest.ts | 18 +++++++------ loadtest/raw-fetch-test.ts | 40 +++++++++++++++++++---------- loadtest/undici-test.ts | 52 +++++++++++++++++++++++--------------- 4 files changed, 81 insertions(+), 50 deletions(-) diff --git a/loadtest/h2-test.ts b/loadtest/h2-test.ts index 7922e3fcd..5da82ed6c 100644 --- a/loadtest/h2-test.ts +++ b/loadtest/h2-test.ts @@ -46,6 +46,11 @@ function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: num } async function main() { + if (!Number.isInteger(NUM_CONNECTIONS) || NUM_CONNECTIONS < 1) { + console.error(`NUM_CONNECTIONS must be a positive integer (got "${process.env.NUM_CONNECTIONS}")`); + process.exit(1); + } + console.log(`HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`); const url = new URL(BASE_URL); @@ -89,13 +94,15 @@ async function main() { console.log(`Connections: ${NUM_CONNECTIONS}`); console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); - console.log(`\nLatency (ms):`); - console.log(` min: ${latencies[0].toFixed(1)}`); - console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); - console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); - console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); - console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); - console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + if (latencies.length > 0) { + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + } console.log(`\nStatus codes:`); for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); } diff --git a/loadtest/loadtest.ts b/loadtest/loadtest.ts index e75775885..3f90264e0 100644 --- a/loadtest/loadtest.ts +++ b/loadtest/loadtest.ts @@ -83,14 +83,16 @@ function printMetrics(results: RequestResult[], wallClockMs: number): void { console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`); console.log(`Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`); console.log(''); - console.log('Latency (ms):'); - console.log(` min: ${latencies[0].toFixed(1)}`); - console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); - console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); - console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); - console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); - console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); - console.log(''); + if (latencies.length > 0) { + console.log('Latency (ms):'); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + console.log(''); + } console.log('Status codes:'); for (const [status, count] of [...statusCounts.entries()].sort()) { console.log(` ${status}: ${count}`); diff --git a/loadtest/raw-fetch-test.ts b/loadtest/raw-fetch-test.ts index c55bb80a9..396590b63 100644 --- a/loadtest/raw-fetch-test.ts +++ b/loadtest/raw-fetch-test.ts @@ -18,9 +18,17 @@ const body = JSON.stringify({ launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 }, }); -function makeRequest(index: number): Promise<{ latencyMs: number; status: number }> { +function makeRequest(index: number): Promise<{ latencyMs: number; status: number | null }> { const start = performance.now(); - return new Promise((resolve, reject) => { + return new Promise((resolve) => { + // Settle exactly once. A request/response error or a mid-stream abort resolves + // with status=null (a failure) rather than hanging or rejecting the whole batch. + let settled = false; + const done = (status: number | null) => { + if (settled) return; + settled = true; + resolve({ latencyMs: performance.now() - start, status }); + }; const url = new URL('/v1/devboxes', BASE_URL); const req = https.request( url, @@ -35,11 +43,12 @@ function makeRequest(index: number): Promise<{ latencyMs: number; status: number }, (res) => { res.resume(); - res.on('end', () => resolve({ latencyMs: performance.now() - start, status: res.statusCode! })); - res.on('error', reject); + res.on('end', () => done(res.statusCode ?? null)); + res.on('aborted', () => done(null)); + res.on('error', () => done(null)); }, ); - req.on('error', reject); + req.on('error', () => done(null)); req.end(body); }); } @@ -57,20 +66,23 @@ async function main() { const wallMs = performance.now() - wallStart; const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); - const statusCounts = new Map(); + const statusCounts = new Map(); for (const r of results) { - statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + const key = r.status != null ? String(r.status) : 'network_error'; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); } console.log(`\nWall clock: ${(wallMs / 1000).toFixed(2)}s`); console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); - console.log(`\nLatency (ms):`); - console.log(` min: ${latencies[0].toFixed(1)}`); - console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); - console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); - console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); - console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); - console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + if (latencies.length > 0) { + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + } console.log(`\nStatus codes:`); for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); } diff --git a/loadtest/undici-test.ts b/loadtest/undici-test.ts index f0c42292e..f0221180e 100644 --- a/loadtest/undici-test.ts +++ b/loadtest/undici-test.ts @@ -44,20 +44,27 @@ async function main() { const wallStart = performance.now(); + // Collect per-request results (never reject): one transient network error must + // not fail-fast the whole batch and discard every other completed measurement. const promises = Array.from({ length: REQUEST_COUNT }, async () => { const start = performance.now(); - const res = await fetch(`${BASE_URL}/v1/devboxes`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${API_KEY}`, - }, - body, - dispatcher, - } as any); - await res.text(); - completed++; - return { latencyMs: performance.now() - start, status: res.status }; + try { + const res = await fetch(`${BASE_URL}/v1/devboxes`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${API_KEY}`, + }, + body, + dispatcher, + } as any); + await res.text(); + return { latencyMs: performance.now() - start, status: res.status as number | null }; + } catch { + return { latencyMs: performance.now() - start, status: null as number | null }; + } finally { + completed++; + } }); const results = await Promise.all(promises); @@ -67,22 +74,25 @@ async function main() { await dispatcher.close(); const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b); - const statusCounts = new Map(); + const statusCounts = new Map(); for (const r of results) { - statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1); + const key = r.status != null ? String(r.status) : 'network_error'; + statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1); } console.log(`\n=== undici Direct Results ===`); console.log(`Requests: ${REQUEST_COUNT}`); console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`); console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`); - console.log(`\nLatency (ms):`); - console.log(` min: ${latencies[0].toFixed(1)}`); - console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); - console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); - console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); - console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); - console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + if (latencies.length > 0) { + console.log(`\nLatency (ms):`); + console.log(` min: ${latencies[0].toFixed(1)}`); + console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`); + console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`); + console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`); + console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`); + console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`); + } console.log(`\nStatus codes:`); for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`); }