Skip to content

Commit f2cd19a

Browse files
Reflexclaude
Reflex
and
claude
committed
fix(loadtest): make comparison scripts robust to failures and empty input
Address CodeAnt review on the loadtest harness: - undici-test / raw-fetch-test: collect per-request results instead of `Promise.all` fail-fast. One transient network error (or a mid-stream response abort in raw-fetch) no longer hangs or discards the whole run; failures are counted as `network_error` in the status breakdown. - raw-fetch-test: settle each request exactly once across end / aborted / error / request-error paths (previously a mid-stream abort could hang the batch indefinitely). - h2-test: reject NUM_CONNECTIONS < 1 up front (previously dereferenced clients[0] === undefined and crashed). - All scripts: guard the latency block against zero results so an empty run doesn't throw on latencies[0]. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 66fe8dc commit f2cd19a

4 files changed

Lines changed: 81 additions & 50 deletions

File tree

‎loadtest/h2-test.ts‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ function sendRequest(client: http2.ClientHttp2Session): Promise<{ latencyMs: num
4646
}
4747

4848
async function main() {
49+
if (!Number.isInteger(NUM_CONNECTIONS) || NUM_CONNECTIONS < 1) {
50+
console.error(`NUM_CONNECTIONS must be a positive integer (got "${process.env.NUM_CONNECTIONS}")`);
51+
process.exit(1);
52+
}
53+
4954
console.log(`HTTP/2 test: ${REQUEST_COUNT} requests, ${NUM_CONNECTIONS} connections to ${BASE_URL}`);
5055

5156
const url = new URL(BASE_URL);
@@ -89,13 +94,15 @@ async function main() {
8994
console.log(`Connections: ${NUM_CONNECTIONS}`);
9095
console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`);
9196
console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`);
92-
console.log(`\nLatency (ms):`);
93-
console.log(` min: ${latencies[0].toFixed(1)}`);
94-
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
95-
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
96-
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
97-
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
98-
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
97+
if (latencies.length > 0) {
98+
console.log(`\nLatency (ms):`);
99+
console.log(` min: ${latencies[0].toFixed(1)}`);
100+
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
101+
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
102+
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
103+
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
104+
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
105+
}
99106
console.log(`\nStatus codes:`);
100107
for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`);
101108
}

‎loadtest/loadtest.ts‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,16 @@ function printMetrics(results: RequestResult[], wallClockMs: number): void {
8383
console.log(`Wall clock: ${(wallClockMs / 1000).toFixed(2)}s`);
8484
console.log(`Throughput: ${(results.length / (wallClockMs / 1000)).toFixed(1)} req/s`);
8585
console.log('');
86-
console.log('Latency (ms):');
87-
console.log(` min: ${latencies[0].toFixed(1)}`);
88-
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
89-
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
90-
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
91-
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
92-
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
93-
console.log('');
86+
if (latencies.length > 0) {
87+
console.log('Latency (ms):');
88+
console.log(` min: ${latencies[0].toFixed(1)}`);
89+
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
90+
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
91+
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
92+
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
93+
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
94+
console.log('');
95+
}
9496
console.log('Status codes:');
9597
for (const [status, count] of [...statusCounts.entries()].sort()) {
9698
console.log(` ${status}: ${count}`);

‎loadtest/raw-fetch-test.ts‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,17 @@ const body = JSON.stringify({
1818
launch_parameters: { resource_size_request: 'SMALL', keep_alive_time_seconds: 300 },
1919
});
2020

21-
function makeRequest(index: number): Promise<{ latencyMs: number; status: number }> {
21+
function makeRequest(index: number): Promise<{ latencyMs: number; status: number | null }> {
2222
const start = performance.now();
23-
return new Promise((resolve, reject) => {
23+
return new Promise((resolve) => {
24+
// Settle exactly once. A request/response error or a mid-stream abort resolves
25+
// with status=null (a failure) rather than hanging or rejecting the whole batch.
26+
let settled = false;
27+
const done = (status: number | null) => {
28+
if (settled) return;
29+
settled = true;
30+
resolve({ latencyMs: performance.now() - start, status });
31+
};
2432
const url = new URL('/v1/devboxes', BASE_URL);
2533
const req = https.request(
2634
url,
@@ -35,11 +43,12 @@ function makeRequest(index: number): Promise<{ latencyMs: number; status: number
3543
},
3644
(res) => {
3745
res.resume();
38-
res.on('end', () => resolve({ latencyMs: performance.now() - start, status: res.statusCode! }));
39-
res.on('error', reject);
46+
res.on('end', () => done(res.statusCode ?? null));
47+
res.on('aborted', () => done(null));
48+
res.on('error', () => done(null));
4049
},
4150
);
42-
req.on('error', reject);
51+
req.on('error', () => done(null));
4352
req.end(body);
4453
});
4554
}
@@ -57,20 +66,23 @@ async function main() {
5766
const wallMs = performance.now() - wallStart;
5867

5968
const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b);
60-
const statusCounts = new Map<number, number>();
69+
const statusCounts = new Map<string, number>();
6170
for (const r of results) {
62-
statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1);
71+
const key = r.status != null ? String(r.status) : 'network_error';
72+
statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1);
6373
}
6474

6575
console.log(`\nWall clock: ${(wallMs / 1000).toFixed(2)}s`);
6676
console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`);
67-
console.log(`\nLatency (ms):`);
68-
console.log(` min: ${latencies[0].toFixed(1)}`);
69-
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
70-
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
71-
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
72-
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
73-
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
77+
if (latencies.length > 0) {
78+
console.log(`\nLatency (ms):`);
79+
console.log(` min: ${latencies[0].toFixed(1)}`);
80+
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
81+
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
82+
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
83+
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
84+
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
85+
}
7486
console.log(`\nStatus codes:`);
7587
for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`);
7688
}

‎loadtest/undici-test.ts‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,27 @@ async function main() {
4444

4545
const wallStart = performance.now();
4646

47+
// Collect per-request results (never reject): one transient network error must
48+
// not fail-fast the whole batch and discard every other completed measurement.
4749
const promises = Array.from({ length: REQUEST_COUNT }, async () => {
4850
const start = performance.now();
49-
const res = await fetch(`${BASE_URL}/v1/devboxes`, {
50-
method: 'POST',
51-
headers: {
52-
'content-type': 'application/json',
53-
authorization: `Bearer ${API_KEY}`,
54-
},
55-
body,
56-
dispatcher,
57-
} as any);
58-
await res.text();
59-
completed++;
60-
return { latencyMs: performance.now() - start, status: res.status };
51+
try {
52+
const res = await fetch(`${BASE_URL}/v1/devboxes`, {
53+
method: 'POST',
54+
headers: {
55+
'content-type': 'application/json',
56+
authorization: `Bearer ${API_KEY}`,
57+
},
58+
body,
59+
dispatcher,
60+
} as any);
61+
await res.text();
62+
return { latencyMs: performance.now() - start, status: res.status as number | null };
63+
} catch {
64+
return { latencyMs: performance.now() - start, status: null as number | null };
65+
} finally {
66+
completed++;
67+
}
6168
});
6269

6370
const results = await Promise.all(promises);
@@ -67,22 +74,25 @@ async function main() {
6774
await dispatcher.close();
6875

6976
const latencies = results.map((r) => r.latencyMs).sort((a, b) => a - b);
70-
const statusCounts = new Map<number, number>();
77+
const statusCounts = new Map<string, number>();
7178
for (const r of results) {
72-
statusCounts.set(r.status, (statusCounts.get(r.status) ?? 0) + 1);
79+
const key = r.status != null ? String(r.status) : 'network_error';
80+
statusCounts.set(key, (statusCounts.get(key) ?? 0) + 1);
7381
}
7482

7583
console.log(`\n=== undici Direct Results ===`);
7684
console.log(`Requests: ${REQUEST_COUNT}`);
7785
console.log(`Wall clock: ${(wallMs / 1000).toFixed(2)}s`);
7886
console.log(`Throughput: ${(REQUEST_COUNT / (wallMs / 1000)).toFixed(1)} req/s`);
79-
console.log(`\nLatency (ms):`);
80-
console.log(` min: ${latencies[0].toFixed(1)}`);
81-
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
82-
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
83-
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
84-
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
85-
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
87+
if (latencies.length > 0) {
88+
console.log(`\nLatency (ms):`);
89+
console.log(` min: ${latencies[0].toFixed(1)}`);
90+
console.log(` p50: ${percentile(latencies, 50).toFixed(1)}`);
91+
console.log(` p90: ${percentile(latencies, 90).toFixed(1)}`);
92+
console.log(` p95: ${percentile(latencies, 95).toFixed(1)}`);
93+
console.log(` p99: ${percentile(latencies, 99).toFixed(1)}`);
94+
console.log(` max: ${latencies[latencies.length - 1].toFixed(1)}`);
95+
}
8696
console.log(`\nStatus codes:`);
8797
for (const [s, c] of [...statusCounts.entries()].sort()) console.log(` ${s}: ${c}`);
8898
}

0 commit comments

Comments
 (0)