Skip to content

Commit b9625b4

Browse files
jrvb-rlclaude
jrvb-rl
and
claude
committed
feat: add HTTP/2 load testing infrastructure
Adds a loadtest/ directory with scripts for benchmarking HTTP/2 transport throughput against the Runloop API. Includes baselines for raw node:http2, undici, and the SDK client at various concurrency levels. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 43fcd45 commit b9625b4

10 files changed

Lines changed: 730 additions & 0 deletions

‎loadtest/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
node_modules/
2+
package-lock.json

‎loadtest/alpn-check.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import tls from "node:tls";
2+
3+
const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro";
4+
const url = new URL(BASE_URL);
5+
6+
console.log(`Checking ALPN for ${url.hostname}:${url.port || 443}`);
7+
8+
const socket = tls.connect(
9+
{
10+
host: url.hostname,
11+
port: parseInt(url.port || "443", 10),
12+
ALPNProtocols: ["h2", "http/1.1"],
13+
servername: url.hostname,
14+
},
15+
() => {
16+
console.log(`Negotiated protocol: ${socket.alpnProtocol}`);
17+
console.log(`TLS version: ${socket.getProtocol()}`);
18+
socket.end();
19+
},
20+
);
21+
22+
socket.on("error", (err) => {
23+
console.error("TLS error:", err.message);
24+
});

‎loadtest/h2-single-conn.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import http2 from "node:http2";
2+
3+
const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro";
4+
const API_KEY = process.env.RUNLOOP_API_KEY!;
5+
6+
const body = JSON.stringify({
7+
blueprint_id: "bp_nonexistent_loadtest_00000",
8+
name: "loadtest-h2s-0",
9+
environment_variables: { TEST_VAR_1: "value_one" },
10+
launch_parameters: { resource_size_request: "SMALL" },
11+
});
12+
13+
function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: number; status: number }> {
14+
return new Promise((resolve, reject) => {
15+
const start = performance.now();
16+
const req = client.request({
17+
":method": "POST",
18+
":path": "/v1/devboxes",
19+
"content-type": "application/json",
20+
authorization: `Bearer ${API_KEY}`,
21+
});
22+
req.on("response", (headers) => {
23+
const status = headers[":status"] as number;
24+
req.on("data", () => {});
25+
req.on("end", () => resolve({ latencyMs: performance.now() - start, status }));
26+
});
27+
req.on("error", reject);
28+
req.end(body);
29+
});
30+
}
31+
32+
async function main() {
33+
const url = new URL(BASE_URL);
34+
const client = http2.connect(url.origin);
35+
await new Promise<void>((resolve, reject) => {
36+
client.on("connect", resolve);
37+
client.on("error", reject);
38+
});
39+
40+
const maxStreams = client.remoteSettings?.maxConcurrentStreams;
41+
console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams}`);
42+
43+
// Warmup
44+
const w = await sendRequest(client);
45+
console.log(`Warmup: status=${w.status}, latency=${w.latencyMs.toFixed(0)}ms`);
46+
47+
// Burst 50 requests on single warmed connection
48+
const count = 50;
49+
console.log(`\nBursting ${count} requests on 1 warmed connection...`);
50+
const wallStart = performance.now();
51+
const results = await Promise.all(Array.from({ length: count }, () => sendRequest(client)));
52+
const wallMs = performance.now() - wallStart;
53+
54+
const lats = results.map((r) => r.latencyMs).sort((a, b) => a - b);
55+
console.log(`${count} requests in ${wallMs.toFixed(0)}ms (${(count / (wallMs / 1000)).toFixed(1)} req/s)`);
56+
console.log(`Latency: min=${lats[0].toFixed(0)}ms p50=${lats[Math.floor(count / 2)].toFixed(0)}ms max=${lats[lats.length - 1].toFixed(0)}ms`);
57+
58+
client.close();
59+
}
60+
61+
main().catch((e) => { console.error(e); process.exit(1); });

‎loadtest/h2-test.ts‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import http2 from "node:http2";
2+
3+
const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "10000", 10);
4+
const NUM_CONNECTIONS = parseInt(process.env.NUM_CONNECTIONS ?? "10", 10);
5+
const BASE_URL = process.env.RUNLOOP_BASE_URL ?? "https://api.runloop.pro";
6+
const API_KEY = process.env.RUNLOOP_API_KEY!;
7+
8+
const body = JSON.stringify({
9+
blueprint_id: "bp_nonexistent_loadtest_00000",
10+
name: "loadtest-h2-0",
11+
environment_variables: { TEST_VAR_1: "value_one", TEST_VAR_2: "value_two" },
12+
metadata: { test_run: "h2", index: "0" },
13+
launch_parameters: { resource_size_request: "SMALL", keep_alive_time_seconds: 300 },
14+
});
15+
16+
function percentile(sorted: number[], p: number): number {
17+
const idx = Math.ceil((p / 100) * sorted.length) - 1;
18+
return sorted[Math.max(0, idx)];
19+
}
20+
21+
function connectH2(origin: string): Promise<http2.ClientHttp2Session> {
22+
return new Promise((resolve, reject) => {
23+
const client = http2.connect(origin);
24+
client.on("connect", () => resolve(client));
25+
client.on("error", reject);
26+
});
27+
}
28+
29+
function sendRequest(
30+
client: http2.ClientHttp2Session,
31+
): Promise<{ latencyMs: number; status: number }> {
32+
return new Promise((resolve, reject) => {
33+
const start = performance.now();
34+
const req = client.request({
35+
":method": "POST",
36+
":path": "/v1/devboxes",
37+
"content-type": "application/json",
38+
authorization: `Bearer ${API_KEY}`,
39+
});
40+
req.on("response", (headers) => {
41+
const status = headers[":status"] as number;
42+
req.on("data", () => {});
43+
req.on("end", () => resolve({ latencyMs: performance.now() - start, status }));
44+
});
45+
req.on("error", reject);
46+
req.end(body);
47+
});
48+
}
49+
50+
async function main() {
51+
console.log(
52+
`HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`,
53+
);
54+
55+
const url = new URL(BASE_URL);
56+
const clients = await Promise.all(
57+
Array.from({ length: NUM_CONNECTIONS }, () => connectH2(url.origin)),
58+
);
59+
60+
const maxStreams = clients[0].remoteSettings?.maxConcurrentStreams;
61+
console.log(`Server MAX_CONCURRENT_STREAMS: ${maxStreams ?? "unknown"}`);
62+
console.log(`${NUM_CONNECTIONS} connections established\n`);
63+
64+
let completed = 0;
65+
const progressTimer = setInterval(() => {
66+
console.log(
67+
` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`,
68+
);
69+
}, 2000);
70+
71+
const wallStart = performance.now();
72+
73+
const promises = Array.from({ length: REQUEST_COUNT }, (_, i) => {
74+
const client = clients[i % NUM_CONNECTIONS];
75+
return sendRequest(client).then((r) => {
76+
completed++;
77+
return r;
78+
});
79+
});
80+
81+
const results = await Promise.all(promises);
82+
const wallMs = performance.now() - wallStart;
83+
clearInterval(progressTimer);
84+
85+
for (const c of clients) c.close();
86+
87+
const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b);
88+
const statusCounts = new Map<number, number>();
89+
for (const r of results) {
90+
statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1);
91+
}
92+
93+
console.log(`\n=== HTTP/2 Results ===`);
94+
console.log(`Requests: ${REQUEST_COUNT}`);
95+
console.log(`Connections: ${NUM_CONNECTIONS}`);
96+
console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`);
97+
console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`);
98+
console.log(`\nLatency (ms):`);
99+
console.log(` min: ${latencies[0].toFixed(1)}`);
100+
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
101+
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
102+
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
103+
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
104+
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
105+
console.log(`\nStatus codes:`);
106+
for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`);
107+
}
108+
109+
main().catch((e) => {
110+
console.error(e);
111+
process.exit(1);
112+
});

‎loadtest/loadtest.ts‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
import { Runloop } from "@runloop/api-client";
2+
3+
const REQUEST_COUNT = parseInt(process.env.REQUEST_COUNT ?? "100000", 10);
4+
const RUNLOOP_BASE_URL = process.env.RUNLOOP_BASE_URL;
5+
const USE_HTTP2 = process.env.USE_HTTP2 === "1";
6+
const PROGRESS_INTERVAL_MS = 2000;
7+
8+
function buildClient(): Runloop {
9+
const opts: ConstructorParameters<typeof Runloop>[0] = {
10+
maxRetries: 0,
11+
timeout: 120_000,
12+
};
13+
if (RUNLOOP_BASE_URL) {
14+
opts.baseURL = RUNLOOP_BASE_URL;
15+
}
16+
if (USE_HTTP2) {
17+
(opts as any).http2 = true;
18+
}
19+
return new Runloop(opts);
20+
}
21+
22+
interface RequestResult {
23+
index: number;
24+
latencyMs: number;
25+
status: number | null;
26+
error: string | null;
27+
}
28+
29+
async function sendRequest(
30+
client: Runloop,
31+
index: number,
32+
runId: string,
33+
): Promise<RequestResult> {
34+
const start = performance.now();
35+
try {
36+
await client.devboxes.create({
37+
blueprint_id: "bp_nonexistent_loadtest_00000",
38+
name: `loadtest-${runId}-${index}`,
39+
environment_variables: {
40+
TEST_VAR_1: "value_one",
41+
TEST_VAR_2: "value_two",
42+
},
43+
metadata: {
44+
test_run: runId,
45+
index: String(index),
46+
},
47+
launch_parameters: {
48+
resource_size_request: "SMALL",
49+
keep_alive_time_seconds: 300,
50+
},
51+
});
52+
return {
53+
index,
54+
latencyMs: performance.now() - start,
55+
status: 200,
56+
error: null,
57+
};
58+
} catch (err: any) {
59+
return {
60+
index,
61+
latencyMs: performance.now() - start,
62+
status: err?.status ?? null,
63+
error: err?.message ?? String(err),
64+
};
65+
}
66+
}
67+
68+
function percentile(sorted: number[], p: number): number {
69+
const idx = Math.ceil((p / 100) * sorted.length) - 1;
70+
return sorted[Math.max(0, idx)];
71+
}
72+
73+
function printMetrics(results: RequestResult[], wallClockMs: number): void {
74+
const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b);
75+
76+
const statusCounts = new Map<string, number>();
77+
for (const r of results) {
78+
const key = r.status != null ? String(r.status) : "network_error";
79+
statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1);
80+
}
81+
82+
console.log("\n=== Load Test Results ===");
83+
console.log(`Requests: ${results.length}`);
84+
console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`);
85+
console.log(
86+
`Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`,
87+
);
88+
console.log("");
89+
console.log("Latency (ms):");
90+
console.log(` min: ${latencies[0].toFixed(1)}`);
91+
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
92+
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
93+
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
94+
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
95+
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
96+
console.log("");
97+
console.log("Status codes:");
98+
for (const [status, count] of [...statusCounts.entries()].sort()) {
99+
console.log(` ${status}: ${count}`);
100+
}
101+
}
102+
103+
async function main(): Promise<void> {
104+
const fdLimit = await checkFileDescriptorLimit();
105+
if (!USE_HTTP2 && fdLimit < 10000) {
106+
console.warn(
107+
`\nWARNING: File descriptor limit is ${fdLimit}. For 100k HTTP/1.1 requests, run:\n` +
108+
` ulimit -n 65536\n` +
109+
`Or use HTTP/2 multiplexing: USE_HTTP2=1\n`,
110+
);
111+
}
112+
113+
const client = buildClient();
114+
const runId = `run-${Date.now()}`;
115+
116+
console.log(`Starting load test: ${REQUEST_COUNT} concurrent requests`);
117+
console.log(`Run ID: ${runId}`);
118+
console.log(`HTTP mode: ${USE_HTTP2 ? "HTTP/2 (undici)" : "HTTP/1.1 (node-fetch)"}`);
119+
console.log(`Base URL: ${RUNLOOP_BASE_URL ?? "(SDK default)"}`);
120+
console.log(`File descriptor limit: ${fdLimit}`);
121+
console.log("");
122+
123+
let completed = 0;
124+
const progressTimer = setInterval(() => {
125+
console.log(
126+
` progress: ${completed}/${REQUEST_COUNT} (${((completed / REQUEST_COUNT) * 100).toFixed(1)}%)`,
127+
);
128+
}, PROGRESS_INTERVAL_MS);
129+
130+
const wallStart = performance.now();
131+
132+
const promises = Array.from({ length: REQUEST_COUNT }, (_, i) =>
133+
sendRequest(client, i, runId).then((result) => {
134+
completed++;
135+
return result;
136+
}),
137+
);
138+
139+
const results = await Promise.all(promises);
140+
141+
const wallClockMs = performance.now() - wallStart;
142+
clearInterval(progressTimer);
143+
144+
printMetrics(results, wallClockMs);
145+
}
146+
147+
async function checkFileDescriptorLimit(): Promise<number> {
148+
try {
149+
const { execSync } = await import("child_process");
150+
return parseInt(execSync("ulimit -n", { encoding: "utf-8" }).trim(), 10);
151+
} catch {
152+
return -1;
153+
}
154+
}
155+
156+
main().catch((err) => {
157+
console.error("Fatal error:", err);
158+
process.exit(1);
159+
});

‎loadtest/package.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"name": "runloop-loadtest",
3+
"private": true,
4+
"type": "module",
5+
"scripts": {
6+
"test": "tsx loadtest.ts"
7+
},
8+
"dependencies": {
9+
"@runloop/api-client": "file:..",
10+
"tsx": "^4.0.0",
11+
"undici": "^7.26.0"
12+
},
13+
"devDependencies": {
14+
"@types/node": "^25.9.3"
15+
}
16+
}

0 commit comments

Comments
 (0)