From 6329254ba121a2b23fcbf072ad713a9c43f221ec Mon Sep 17 00:00:00 2001 From: Yoon Park Date: Fri, 10 Jul 2026 17:02:17 -0700 Subject: [PATCH 1/6] feat: add HTTP/2 load testing infrastructure Co-Authored-By: Claude Sonnet 4.6 --- loadtest/.gitignore | 2 + loadtest/README.md | 58 +++++++++++++++ loadtest/alpn_check.py | 22 ++++++ loadtest/h2_single_conn.py | 60 ++++++++++++++++ loadtest/h2_test.py | 110 +++++++++++++++++++++++++++++ loadtest/loadtest.py | 140 +++++++++++++++++++++++++++++++++++++ loadtest/raw_fetch_test.py | 98 ++++++++++++++++++++++++++ 7 files changed, 490 insertions(+) create mode 100644 loadtest/.gitignore create mode 100644 loadtest/README.md create mode 100644 loadtest/alpn_check.py create mode 100644 loadtest/h2_single_conn.py create mode 100644 loadtest/h2_test.py create mode 100644 loadtest/loadtest.py create mode 100644 loadtest/raw_fetch_test.py diff --git a/loadtest/.gitignore b/loadtest/.gitignore new file mode 100644 index 000000000..7a60b85e1 --- /dev/null +++ b/loadtest/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/loadtest/README.md b/loadtest/README.md new file mode 100644 index 000000000..cf3e4669d --- /dev/null +++ b/loadtest/README.md @@ -0,0 +1,58 @@ +# Load tests + +Manual transport comparison scripts for the Runloop Python SDK. These hit the +real API and are not part of `pytest`; run on demand. + +## End-to-end transport comparison (real API) + +All scripts require `RUNLOOP_API_KEY`. Set `RUNLOOP_BASE_URL` to override the +default endpoint (`https://api.runloop.ai`). + +Every script sends `devboxes.create` requests against a **deliberately +nonexistent blueprint** (`bp_nonexistent_loadtest_00000`), so every request +fails fast server-side (HTTP `400`) and **no devboxes are created** — isolating +client + server _request handling_ from provisioning. + +| Script | Transport under test | +| --- | --- | +| `loadtest.py` | The **SDK** itself (`AsyncRunloop`). `USE_HTTP2=0` → HTTP/1.1; default / `USE_HTTP2=1` → HTTP/2 (shared httpx pool). Always runs against the installed package in this checkout. | +| `h2_test.py` | Raw `httpx` HTTP/2, bypassing the SDK. Configurable connection count. | +| `h2_single_conn.py` | Raw `httpx` HTTP/2 on a single warmed connection (50-request burst). | +| `raw_fetch_test.py` | Raw `httpx` HTTP/1.1 keep-alive baseline. | +| `alpn_check.py` | Confirms the origin negotiates `h2` via TLS ALPN. | + +The raw-transport probes compare httpx HTTP/2 multiplexing against HTTP/1.1 +directly — the same comparison that motivates the SDK's shared HTTP/2 pool. +They're kept so the comparison stays reproducible. + +```sh +# Install the SDK from this checkout (editable): +cd /path/to/api-client-python && uv sync + +# SDK: HTTP/2 (default) vs HTTP/1.1, 2000-request burst +source ~/env && REQUEST_COUNT=2000 uv run python loadtest/loadtest.py # HTTP/2 +source ~/env && REQUEST_COUNT=2000 USE_HTTP2=0 uv run python loadtest/loadtest.py # HTTP/1.1 + +# Raw httpx HTTP/2 vs HTTP/1.1 comparison +source ~/env && uv run python loadtest/h2_test.py +source ~/env && uv run python loadtest/raw_fetch_test.py + +# Single-connection burst +source ~/env && uv run python loadtest/h2_single_conn.py + +# ALPN check +source ~/env && uv run python loadtest/alpn_check.py +``` + +HTTP/1.1 opens a socket per in-flight request; for large bursts raise the +file-descriptor limit (`ulimit -n 65536`) or keep `REQUEST_COUNT` small. + +## Environment variables + +| Variable | Default | Description | +| --- | --- | --- | +| `RUNLOOP_API_KEY` | *(required)* | API key | +| `RUNLOOP_BASE_URL` | `https://api.runloop.ai` | Override API endpoint | +| `REQUEST_COUNT` | `100000` (`loadtest.py`) / `10000` (`h2_test.py`) / `500` (`raw_fetch_test.py`) | Total requests | +| `NUM_CONNECTIONS` | `10` (`h2_test.py`) / `20` (`raw_fetch_test.py`) | Parallel connections | +| `USE_HTTP2` | `1` | `0` to force HTTP/1.1 in `loadtest.py` | diff --git a/loadtest/alpn_check.py b/loadtest/alpn_check.py new file mode 100644 index 000000000..e0ab701cd --- /dev/null +++ b/loadtest/alpn_check.py @@ -0,0 +1,22 @@ +"""Confirm the origin negotiates h2 via TLS ALPN.""" + +from __future__ import annotations + +import os +import ssl +import socket + +BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai") +url_parts = BASE_URL.split("://", 1) +host = url_parts[1].split("/")[0] if len(url_parts) > 1 else url_parts[0] +port = 443 + +print(f"Checking ALPN for {host}:{port}") + +ctx = ssl.create_default_context() +ctx.set_alpn_protocols(["h2", "http/1.1"]) + +with socket.create_connection((host, port)) as sock: + with ctx.wrap_socket(sock, server_hostname=host) as tls: + print(f"Negotiated protocol: {tls.selected_alpn_protocol()}") + print(f"TLS version: {tls.version()}") diff --git a/loadtest/h2_single_conn.py b/loadtest/h2_single_conn.py new file mode 100644 index 000000000..fcaec1bc9 --- /dev/null +++ b/loadtest/h2_single_conn.py @@ -0,0 +1,60 @@ +"""Raw httpx HTTP/2 single-connection burst: 50 requests on one warmed connection.""" + +from __future__ import annotations + +import asyncio +import os +import time + +import httpx + +BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai") +API_KEY = os.environ["RUNLOOP_API_KEY"] + +BODY = { + "blueprint_id": "bp_nonexistent_loadtest_00000", + "name": "loadtest-h2s-0", + "environment_variables": {"TEST_VAR_1": "value_one"}, + "launch_parameters": {"resource_size_request": "SMALL"}, +} + + +async def send_request(client: httpx.AsyncClient) -> dict[str, object]: + start = time.perf_counter() + response = await client.post( + "/v1/devboxes", + json=BODY, + headers={"authorization": f"Bearer {API_KEY}"}, + ) + return {"latency_ms": (time.perf_counter() - start) * 1000, "status": response.status_code} + + +async def main() -> None: + client = httpx.AsyncClient( + base_url=BASE_URL, + http2=True, + limits=httpx.Limits(max_connections=1, max_keepalive_connections=1), + timeout=httpx.Timeout(120.0), + ) + + # Warmup + w = await send_request(client) + print(f"Warmup: status={w['status']}, latency={w['latency_ms']:.0f}ms") + + count = 50 + print(f"\nBursting {count} requests on 1 warmed connection...") + wall_start = time.perf_counter() + results = await asyncio.gather(*(send_request(client) for _ in range(count))) + wall_ms = (time.perf_counter() - wall_start) * 1000 + + await client.aclose() + + lats = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + print(f"{count} requests in {wall_ms:.0f}ms ({count / (wall_ms / 1000):.1f} req/s)") + print( + f"Latency: min={lats[0]:.0f}ms p50={lats[count // 2]:.0f}ms max={lats[-1]:.0f}ms" + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/loadtest/h2_test.py b/loadtest/h2_test.py new file mode 100644 index 000000000..142310b1c --- /dev/null +++ b/loadtest/h2_test.py @@ -0,0 +1,110 @@ +"""Raw httpx HTTP/2 load test: REQUEST_COUNT requests across NUM_CONNECTIONS connections.""" + +from __future__ import annotations + +import asyncio +import math +import os +import time + +import httpx + +REQUEST_COUNT = int(os.environ.get("REQUEST_COUNT", "10000")) +NUM_CONNECTIONS = int(os.environ.get("NUM_CONNECTIONS", "10")) +BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai") +API_KEY = os.environ["RUNLOOP_API_KEY"] + +BODY = { + "blueprint_id": "bp_nonexistent_loadtest_00000", + "name": "loadtest-h2-0", + "environment_variables": {"TEST_VAR_1": "value_one", "TEST_VAR_2": "value_two"}, + "metadata": {"test_run": "h2", "index": "0"}, + "launch_parameters": {"resource_size_request": "SMALL", "keep_alive_time_seconds": 300}, +} + + +def percentile(sorted_vals: list[float], p: float) -> float: + idx = math.ceil(p / 100 * len(sorted_vals)) - 1 + return sorted_vals[max(0, idx)] + + +async def send_request(client: httpx.AsyncClient) -> dict[str, object]: + start = time.perf_counter() + response = await client.post( + "/v1/devboxes", + json=BODY, + headers={"authorization": f"Bearer {API_KEY}"}, + ) + return {"latency_ms": (time.perf_counter() - start) * 1000, "status": response.status_code} + + +async def main() -> None: + if NUM_CONNECTIONS < 1: + print(f'NUM_CONNECTIONS must be a positive integer (got "{os.environ.get("NUM_CONNECTIONS")}")') + return + + print(f"HTTP/2 test: {REQUEST_COUNT} requests, {NUM_CONNECTIONS} connections to {BASE_URL}") + + # One client per logical connection — each capped at 1 connection so requests + # are distributed across NUM_CONNECTIONS distinct HTTP/2 sessions. + clients = [ + httpx.AsyncClient( + base_url=BASE_URL, + http2=True, + limits=httpx.Limits(max_connections=1, max_keepalive_connections=1), + timeout=httpx.Timeout(120.0), + ) + for _ in range(NUM_CONNECTIONS) + ] + + print(f"{NUM_CONNECTIONS} connections established\n") + + completed = 0 + + async def progress_printer() -> None: + while True: + await asyncio.sleep(2) + pct = completed / REQUEST_COUNT * 100 + print(f" progress: {completed}/{REQUEST_COUNT} ({pct:.1f}%)") + + async def wrapped(idx: int) -> dict[str, object]: + nonlocal completed + r = await send_request(clients[idx % NUM_CONNECTIONS]) + completed += 1 + return r + + wall_start = time.perf_counter() + progress_task = asyncio.create_task(progress_printer()) + results = await asyncio.gather(*(wrapped(i) for i in range(REQUEST_COUNT))) + progress_task.cancel() + wall_ms = (time.perf_counter() - wall_start) * 1000 + + for c in clients: + await c.aclose() + + latencies = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + status_counts: dict[int, int] = {} + for r in results: + s = int(r["status"]) # type: ignore[arg-type] + status_counts[s] = status_counts.get(s, 0) + 1 + + print(f"\n=== HTTP/2 Results ===") + print(f"Requests: {REQUEST_COUNT}") + print(f"Connections: {NUM_CONNECTIONS}") + print(f"Wall clock: {wall_ms / 1000:.2f}s") + print(f"Throughput: {REQUEST_COUNT / (wall_ms / 1000):.1f} req/s") + if latencies: + print("\nLatency (ms):") + print(f" min: {latencies[0]:.1f}") + print(f" p50: {percentile(latencies, 50):.1f}") + print(f" p90: {percentile(latencies, 90):.1f}") + print(f" p95: {percentile(latencies, 95):.1f}") + print(f" p99: {percentile(latencies, 99):.1f}") + print(f" max: {latencies[-1]:.1f}") + print("\nStatus codes:") + for s, c_count in sorted(status_counts.items()): + print(f" {s}: {c_count}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/loadtest/loadtest.py b/loadtest/loadtest.py new file mode 100644 index 000000000..45337f232 --- /dev/null +++ b/loadtest/loadtest.py @@ -0,0 +1,140 @@ +"""SDK load test: fires devboxes.create bursts via AsyncRunloop. + +Imports the installed package from this checkout so the benchmark always +exercises the exact code here — not a separately published build. + +USE_HTTP2=0 → HTTP/1.1 (httpx with http2=False) +default → HTTP/2 (shared httpx pool with http2=True) +REQUEST_COUNT defaults to 100 000. +""" + +from __future__ import annotations + +import asyncio +import math +import os +import time + +import httpx + +from runloop_api_client import AsyncRunloop + +REQUEST_COUNT = int(os.environ.get("REQUEST_COUNT", "100000")) +RUNLOOP_BASE_URL = os.environ.get("RUNLOOP_BASE_URL") +# HTTP/2 is the SDK default; set USE_HTTP2=0 to benchmark HTTP/1.1. +USE_HTTP2 = os.environ.get("USE_HTTP2", "1") == "1" +PROGRESS_INTERVAL = 2.0 + + +def build_client() -> AsyncRunloop: + kwargs: dict[str, object] = {"max_retries": 0, "timeout": 120.0} + if RUNLOOP_BASE_URL: + kwargs["base_url"] = RUNLOOP_BASE_URL + if not USE_HTTP2: + # A custom http_client disables the shared HTTP/2 pool. + # Set http2: false explicitly — omitting it would select HTTP/2. + kwargs["http_client"] = httpx.AsyncClient( + http2=False, + limits=httpx.Limits(max_connections=None, max_keepalive_connections=100), + timeout=httpx.Timeout(120.0), + ) + return AsyncRunloop(**kwargs) # type: ignore[arg-type] + + +async def send_request(client: AsyncRunloop, index: int, run_id: str) -> dict[str, object]: + start = time.perf_counter() + status: int | None = None + try: + await client.devboxes.create( + blueprint_id="bp_nonexistent_loadtest_00000", + name=f"loadtest-{run_id}-{index}", + environment_variables={"TEST_VAR_1": "value_one", "TEST_VAR_2": "value_two"}, + metadata={"test_run": run_id, "index": str(index)}, + launch_parameters={"resource_size_request": "SMALL", "keep_alive_time_seconds": 300}, + ) + status = 200 + except Exception as exc: + status = getattr(exc, "status_code", None) + return {"index": index, "latency_ms": (time.perf_counter() - start) * 1000, "status": status} + + +def percentile(sorted_vals: list[float], p: float) -> float: + idx = math.ceil(p / 100 * len(sorted_vals)) - 1 + return sorted_vals[max(0, idx)] + + +def print_metrics(results: list[dict[str, object]], wall_ms: float) -> None: + latencies = sorted(float(r["latency_ms"]) for r in results) # type: ignore[arg-type] + status_counts: dict[str, int] = {} + for r in results: + key = str(r["status"]) if r["status"] is not None else "network_error" + status_counts[key] = status_counts.get(key, 0) + 1 + + print("\n=== Load Test Results ===") + print(f"Requests: {len(results)}") + print(f"Wall clock: {wall_ms / 1000:.2f}s") + print(f"Throughput: {len(results) / (wall_ms / 1000):.1f} req/s") + if latencies: + print("\nLatency (ms):") + print(f" min: {latencies[0]:.1f}") + print(f" p50: {percentile(latencies, 50):.1f}") + print(f" p90: {percentile(latencies, 90):.1f}") + print(f" p95: {percentile(latencies, 95):.1f}") + print(f" p99: {percentile(latencies, 99):.1f}") + print(f" max: {latencies[-1]:.1f}") + print("\nStatus codes:") + for s, c in sorted(status_counts.items()): + print(f" {s}: {c}") + + +async def main() -> None: + fd_limit: int | None = None + try: + import resource as _resource + + fd_limit = _resource.getrlimit(_resource.RLIMIT_NOFILE)[0] + except Exception: + pass + + if not USE_HTTP2 and fd_limit is not None and fd_limit < 10000: + print(f"\nWARNING: File descriptor limit is {fd_limit}. For large HTTP/1.1 bursts, run:") + print(" ulimit -n 65536") + print("Or use HTTP/2 multiplexing: USE_HTTP2=1\n") + + client = build_client() + run_id = f"run-{int(time.time())}" + + print(f"Starting load test: {REQUEST_COUNT} concurrent requests") + print(f"Run ID: {run_id}") + print(f"HTTP mode: {'HTTP/2 (shared httpx pool)' if USE_HTTP2 else 'HTTP/1.1 (httpx http2=False)'}") + print(f"Base URL: {RUNLOOP_BASE_URL or '(SDK default)'}") + if fd_limit is not None: + print(f"FD limit: {fd_limit}") + print() + + completed = 0 + + async def progress_printer() -> None: + while True: + await asyncio.sleep(PROGRESS_INTERVAL) + pct = completed / REQUEST_COUNT * 100 + print(f" progress: {completed}/{REQUEST_COUNT} ({pct:.1f}%)") + + async def wrapped(idx: int) -> dict[str, object]: + nonlocal completed + r = await send_request(client, idx, run_id) + completed += 1 + return r + + wall_start = time.perf_counter() + progress_task = asyncio.create_task(progress_printer()) + results = list(await asyncio.gather(*(wrapped(i) for i in range(REQUEST_COUNT)))) + progress_task.cancel() + wall_ms = (time.perf_counter() - wall_start) * 1000 + + await client.close() + print_metrics(results, wall_ms) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/loadtest/raw_fetch_test.py b/loadtest/raw_fetch_test.py new file mode 100644 index 000000000..542ece3d7 --- /dev/null +++ b/loadtest/raw_fetch_test.py @@ -0,0 +1,98 @@ +"""Raw httpx HTTP/1.1 keep-alive baseline.""" + +from __future__ import annotations + +import asyncio +import math +import os +import time + +import httpx + +REQUEST_COUNT = int(os.environ.get("REQUEST_COUNT", "500")) +NUM_CONNECTIONS = int(os.environ.get("NUM_CONNECTIONS", "20")) +BASE_URL = os.environ.get("RUNLOOP_BASE_URL", "https://api.runloop.ai") +API_KEY = os.environ["RUNLOOP_API_KEY"] + +BODY = { + "blueprint_id": "bp_nonexistent_loadtest_00000", + "name": "loadtest-raw-0", + "environment_variables": {"TEST_VAR_1": "value_one", "TEST_VAR_2": "value_two"}, + "metadata": {"test_run": "raw", "index": "0"}, + "launch_parameters": {"resource_size_request": "SMALL", "keep_alive_time_seconds": 300}, +} + + +def percentile(sorted_vals: list[float], p: float) -> float: + idx = math.ceil(p / 100 * len(sorted_vals)) - 1 + return sorted_vals[max(0, idx)] + + +async def main() -> None: + print(f"HTTP/1.1 test: {REQUEST_COUNT} requests, {NUM_CONNECTIONS} keep-alive connections to {BASE_URL}") + + client = httpx.AsyncClient( + base_url=BASE_URL, + http2=False, + limits=httpx.Limits(max_connections=NUM_CONNECTIONS, max_keepalive_connections=NUM_CONNECTIONS), + timeout=httpx.Timeout(120.0), + ) + + completed = 0 + + async def progress_printer() -> None: + while True: + await asyncio.sleep(2) + pct = completed / REQUEST_COUNT * 100 + print(f" progress: {completed}/{REQUEST_COUNT} ({pct:.1f}%)") + + async def send_one() -> dict[str, object]: + nonlocal completed + start = time.perf_counter() + status: int | None = None + try: + response = await client.post( + "/v1/devboxes", + json=BODY, + headers={"authorization": f"Bearer {API_KEY}"}, + ) + status = response.status_code + except Exception: + pass + finally: + completed += 1 + return {"latency_ms": (time.perf_counter() - start) * 1000, "status": status} + + wall_start = time.perf_counter() + progress_task = asyncio.create_task(progress_printer()) + results = await asyncio.gather(*(send_one() for _ in range(REQUEST_COUNT))) + progress_task.cancel() + wall_ms = (time.perf_counter() - wall_start) * 1000 + + await client.aclose() + + latencies = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + status_counts: dict[str, int] = {} + for r in results: + key = str(r["status"]) if r["status"] is not None else "network_error" + status_counts[key] = status_counts.get(key, 0) + 1 + + print(f"\n=== HTTP/1.1 Results ===") + print(f"Requests: {REQUEST_COUNT}") + print(f"Wall clock: {wall_ms / 1000:.2f}s") + print(f"Throughput: {REQUEST_COUNT / (wall_ms / 1000):.1f} req/s") + if latencies: + print("\nLatency (ms):") + print(f" min: {latencies[0]:.1f}") + print(f" p50: {percentile(latencies, 50):.1f}") + print(f" p90: {percentile(latencies, 90):.1f}") + print(f" p95: {percentile(latencies, 95):.1f}") + print(f" p99: {percentile(latencies, 99):.1f}") + print(f" max: {latencies[-1]:.1f}") + print("\nStatus codes:") + for s, c_count in sorted(status_counts.items()): + print(f" {s}: {c_count}") + + +if __name__ == "__main__": + asyncio.run(main()) From dd5db46204984ddd3d056704a1c9b70144adef9b Mon Sep 17 00:00:00 2001 From: Yoon Park Date: Fri, 10 Jul 2026 17:10:52 -0700 Subject: [PATCH 2/6] fix(loadtest): annotate latencies as list[float] to satisfy pyright --- loadtest/h2_test.py | 2 +- loadtest/raw_fetch_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loadtest/h2_test.py b/loadtest/h2_test.py index 142310b1c..8237fa92d 100644 --- a/loadtest/h2_test.py +++ b/loadtest/h2_test.py @@ -82,7 +82,7 @@ async def wrapped(idx: int) -> dict[str, object]: for c in clients: await c.aclose() - latencies = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + latencies: list[float] = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] status_counts: dict[int, int] = {} for r in results: s = int(r["status"]) # type: ignore[arg-type] diff --git a/loadtest/raw_fetch_test.py b/loadtest/raw_fetch_test.py index 542ece3d7..803777520 100644 --- a/loadtest/raw_fetch_test.py +++ b/loadtest/raw_fetch_test.py @@ -71,7 +71,7 @@ async def send_one() -> dict[str, object]: await client.aclose() - latencies = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + latencies: list[float] = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] status_counts: dict[str, int] = {} for r in results: key = str(r["status"]) if r["status"] is not None else "network_error" From 186d0e9e742f4d0e82cafeb3be78d8ce996fbca8 Mon Sep 17 00:00:00 2001 From: Yoon Park Date: Fri, 10 Jul 2026 17:14:51 -0700 Subject: [PATCH 3/6] fix(loadtest): use cast() to satisfy both pyright and mypy --- loadtest/h2_single_conn.py | 3 ++- loadtest/h2_test.py | 5 +++-- loadtest/raw_fetch_test.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/loadtest/h2_single_conn.py b/loadtest/h2_single_conn.py index fcaec1bc9..34c1c3141 100644 --- a/loadtest/h2_single_conn.py +++ b/loadtest/h2_single_conn.py @@ -5,6 +5,7 @@ import asyncio import os import time +from typing import cast import httpx @@ -49,7 +50,7 @@ async def main() -> None: await client.aclose() - lats = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + lats: list[float] = sorted(cast(float, r["latency_ms"]) for r in results) print(f"{count} requests in {wall_ms:.0f}ms ({count / (wall_ms / 1000):.1f} req/s)") print( f"Latency: min={lats[0]:.0f}ms p50={lats[count // 2]:.0f}ms max={lats[-1]:.0f}ms" diff --git a/loadtest/h2_test.py b/loadtest/h2_test.py index 8237fa92d..81944c359 100644 --- a/loadtest/h2_test.py +++ b/loadtest/h2_test.py @@ -6,6 +6,7 @@ import math import os import time +from typing import cast import httpx @@ -82,10 +83,10 @@ async def wrapped(idx: int) -> dict[str, object]: for c in clients: await c.aclose() - latencies: list[float] = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + latencies: list[float] = sorted(cast(float, r["latency_ms"]) for r in results) status_counts: dict[int, int] = {} for r in results: - s = int(r["status"]) # type: ignore[arg-type] + s = cast(int, r["status"]) status_counts[s] = status_counts.get(s, 0) + 1 print(f"\n=== HTTP/2 Results ===") diff --git a/loadtest/raw_fetch_test.py b/loadtest/raw_fetch_test.py index 803777520..617a13b78 100644 --- a/loadtest/raw_fetch_test.py +++ b/loadtest/raw_fetch_test.py @@ -6,6 +6,7 @@ import math import os import time +from typing import cast import httpx @@ -71,7 +72,7 @@ async def send_one() -> dict[str, object]: await client.aclose() - latencies: list[float] = sorted(r["latency_ms"] for r in results) # type: ignore[arg-type] + latencies: list[float] = sorted(cast(float, r["latency_ms"]) for r in results) status_counts: dict[str, int] = {} for r in results: key = str(r["status"]) if r["status"] is not None else "network_error" From 3b51665e736e56e8b15c8d3f2c10a7a347e09fca Mon Sep 17 00:00:00 2001 From: Yoon Park Date: Fri, 10 Jul 2026 17:16:50 -0700 Subject: [PATCH 4/6] fix(loadtest): suppress T201 for loadtest/ and fix import order --- loadtest/h2_single_conn.py | 2 +- loadtest/h2_test.py | 4 ++-- loadtest/loadtest.py | 4 ++-- loadtest/raw_fetch_test.py | 4 ++-- pyproject.toml | 1 + 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/loadtest/h2_single_conn.py b/loadtest/h2_single_conn.py index 34c1c3141..fc79dcc4f 100644 --- a/loadtest/h2_single_conn.py +++ b/loadtest/h2_single_conn.py @@ -2,9 +2,9 @@ from __future__ import annotations -import asyncio import os import time +import asyncio from typing import cast import httpx diff --git a/loadtest/h2_test.py b/loadtest/h2_test.py index 81944c359..5295a2723 100644 --- a/loadtest/h2_test.py +++ b/loadtest/h2_test.py @@ -2,10 +2,10 @@ from __future__ import annotations -import asyncio -import math import os +import math import time +import asyncio from typing import cast import httpx diff --git a/loadtest/loadtest.py b/loadtest/loadtest.py index 45337f232..c7bac803c 100644 --- a/loadtest/loadtest.py +++ b/loadtest/loadtest.py @@ -10,10 +10,10 @@ from __future__ import annotations -import asyncio -import math import os +import math import time +import asyncio import httpx diff --git a/loadtest/raw_fetch_test.py b/loadtest/raw_fetch_test.py index 617a13b78..7400f46b1 100644 --- a/loadtest/raw_fetch_test.py +++ b/loadtest/raw_fetch_test.py @@ -2,10 +2,10 @@ from __future__ import annotations -import asyncio -import math import os +import math import time +import asyncio from typing import cast import httpx diff --git a/pyproject.toml b/pyproject.toml index 7362e5b07..c160c0fa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -256,3 +256,4 @@ known-first-party = ["runloop_api_client", "tests"] "scripts/**.py" = ["T201", "T203"] "tests/**.py" = ["T201", "T203"] "examples/**.py" = ["T201", "T203"] +"loadtest/**.py" = ["T201", "T203"] From 3d30178fe354f4c08d74a331a60a0951bae268eb Mon Sep 17 00:00:00 2001 From: Yoon Park Date: Fri, 10 Jul 2026 17:18:30 -0700 Subject: [PATCH 5/6] fix(loadtest): apply ruff format --- loadtest/h2_single_conn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/loadtest/h2_single_conn.py b/loadtest/h2_single_conn.py index fc79dcc4f..8df712f19 100644 --- a/loadtest/h2_single_conn.py +++ b/loadtest/h2_single_conn.py @@ -52,9 +52,7 @@ async def main() -> None: lats: list[float] = sorted(cast(float, r["latency_ms"]) for r in results) print(f"{count} requests in {wall_ms:.0f}ms ({count / (wall_ms / 1000):.1f} req/s)") - print( - f"Latency: min={lats[0]:.0f}ms p50={lats[count // 2]:.0f}ms max={lats[-1]:.0f}ms" - ) + print(f"Latency: min={lats[0]:.0f}ms p50={lats[count // 2]:.0f}ms max={lats[-1]:.0f}ms") if __name__ == "__main__": From 19f695ef7b18abf23f2f92d14f808387aa2facb5 Mon Sep 17 00:00:00 2001 From: Reflex Date: Sat, 11 Jul 2026 08:00:52 +0000 Subject: [PATCH 6/6] feat(loadtest): capture and report per-request errors Match the TS loadtest's RequestResult by recording the exception message per request, and print a breakdown of the opaque network_error bucket so transport-level failures (timeouts, resets, pool exhaustion) are diagnosable. Co-Authored-By: Claude Opus 4.8 --- loadtest/loadtest.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/loadtest/loadtest.py b/loadtest/loadtest.py index c7bac803c..9db9cc80e 100644 --- a/loadtest/loadtest.py +++ b/loadtest/loadtest.py @@ -44,6 +44,7 @@ def build_client() -> AsyncRunloop: async def send_request(client: AsyncRunloop, index: int, run_id: str) -> dict[str, object]: start = time.perf_counter() status: int | None = None + error: str | None = None try: await client.devboxes.create( blueprint_id="bp_nonexistent_loadtest_00000", @@ -55,7 +56,13 @@ async def send_request(client: AsyncRunloop, index: int, run_id: str) -> dict[st status = 200 except Exception as exc: status = getattr(exc, "status_code", None) - return {"index": index, "latency_ms": (time.perf_counter() - start) * 1000, "status": status} + error = str(exc) or type(exc).__name__ + return { + "index": index, + "latency_ms": (time.perf_counter() - start) * 1000, + "status": status, + "error": error, + } def percentile(sorted_vals: list[float], p: float) -> float: @@ -86,6 +93,18 @@ def print_metrics(results: list[dict[str, object]], wall_ms: float) -> None: for s, c in sorted(status_counts.items()): print(f" {s}: {c}") + # Break down the opaque "network_error" bucket by exception message so + # transport-level failures (timeouts, resets, pool exhaustion) are diagnosable. + error_counts: dict[str, int] = {} + for r in results: + if r["status"] is None and r["error"] is not None: + msg = str(r["error"]) + error_counts[msg] = error_counts.get(msg, 0) + 1 + if error_counts: + print("\nErrors (network_error breakdown):") + for msg, c in sorted(error_counts.items(), key=lambda kv: kv[1], reverse=True): + print(f" {c}x {msg}") + async def main() -> None: fd_limit: int | None = None