Skip to content

Commit 29ec7ea

Browse files
podocarpsohankshirsagar
podocarp
and
sohankshirsagar
authored
Benchmarks (#28)
* initial benchmarks * clean up benchmarks, and add results * merge all benchmarks into one file * make sure benchmarks actually run tusk sdk * fix transforms using http instead of fetch * add new results * fix formatting * update findings * add profiling * added script to compare results * fix compare script * remove results * try adding memory logging * fix merge conflict * revert some files to be same as main * output to json and rewrite compare script * simplify resource collection a lot, and print cpu stats as well * cleanup * super cleanup, fix lots of horrible things * remove comments * add more to readme * try fixing build --------- Co-authored-by: Sohan Kshirsagar <sohan.kshirsagar@gmail.com>
1 parent 1252bf7 commit 29ec7ea

17 files changed

Lines changed: 1618 additions & 144 deletions

‎.tusk/config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
service:
2-
id: "d7fd9519-3151-4638-b59d-8da8f6d63514" # set your own
2+
id: "d7fd9519-3151-4638-b59d-8da8f6d63514" # set your own
33
name: "tusk-backend"
44
port: 9000
55
start:

‎benchmarks/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
results
2+
profile/results
3+
.benchmark-traces

‎benchmarks/README.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Benchmarks
2+
3+
These are some benchmarks.
4+
Each benchmark is actually just a AVA test suite that uses tinybench for
5+
measurements and formatting.
6+
At the start of each test suite, we launch a server in `server/test-server.ts`.
7+
Interestingly, it is this server that does all the work rather than the actual
8+
benchmark files.
9+
10+
## Usage
11+
12+
You can run all tests and get a summary with
13+
```
14+
npm run test:bench
15+
```
16+
You can disable memory monitoring with
17+
```
18+
BENCHMARK_ENABLE_MEMORY=false npm run test:bench
19+
```
20+
Memory monitoring introduces a non-trivial impact to CPU, roughly around 10% or
21+
so.
22+
23+
The results are placed in the `results` folder, and there's a
24+
`compare-benchmarks.ts` script that is run automatically and summarizes the data
25+
in a markdown table; if you want to process the data further this can be a good
26+
starting point.

‎benchmarks/bench/common.ts‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
import test from "ava";
2+
import { Bench, hrtimeNow } from "tinybench";
3+
import { TestServer } from "../server/test-server";
4+
import { createBenchmarkRunResult, persistBenchmarkResult } from "./result-utils";
5+
import { ResourceMonitor } from "./resource-monitor";
6+
7+
let server: TestServer;
8+
let serverUrl: string;
9+
10+
function main(testName: string = "SDK Active") {
11+
test.before(async () => {
12+
server = new TestServer();
13+
const info = await server.start();
14+
serverUrl = info.url;
15+
});
16+
17+
test.after.always(async () => {
18+
if (server) {
19+
await server.stop();
20+
}
21+
});
22+
23+
test.serial(testName, async (t) => {
24+
t.timeout(600_000);
25+
26+
const enableMemoryTracking = process.env.BENCHMARK_ENABLE_MEMORY !== "false";
27+
const resourceMonitor = new ResourceMonitor({
28+
intervalMs: 100,
29+
enableMemoryTracking,
30+
});
31+
32+
const bench = new Bench({
33+
time: 10000,
34+
warmup: false,
35+
now: hrtimeNow,
36+
});
37+
38+
let currentTaskName: string | null = null;
39+
40+
bench.add(
41+
`High CPU: POST /api/compute-hash (${testName})`,
42+
async () => {
43+
const response = await fetch(`${serverUrl}/api/compute-hash`, {
44+
method: "POST",
45+
headers: { "Content-Type": "application/json" },
46+
body: JSON.stringify({ data: "sensitive-data-to-hash", iterations: 1000 }),
47+
});
48+
if (!response.ok) {
49+
throw new Error(`HTTP error! status: ${response.status}`);
50+
}
51+
await response.json();
52+
},
53+
{
54+
beforeAll: () => {
55+
const taskName = `High CPU: POST /api/compute-hash (${testName})`;
56+
console.log("Task starting:", taskName);
57+
resourceMonitor.startTask(taskName);
58+
currentTaskName = taskName;
59+
},
60+
afterAll: () => {
61+
console.log("Task ending:", currentTaskName);
62+
resourceMonitor.endTask();
63+
currentTaskName = null;
64+
},
65+
},
66+
);
67+
68+
bench.add(
69+
`High IO, Low CPU: POST /api/io-bound (${testName})`,
70+
async () => {
71+
const response = await fetch(`${serverUrl}/api/io-bound`, {
72+
method: "POST",
73+
headers: { "Content-Type": "application/json" },
74+
body: JSON.stringify({ jobs: 5, delayMs: 5 }),
75+
});
76+
if (!response.ok) {
77+
throw new Error(`HTTP error! status: ${response.status}`);
78+
}
79+
await response.json();
80+
},
81+
{
82+
beforeAll: () => {
83+
const taskName = `High IO, Low CPU: POST /api/io-bound (${testName})`;
84+
console.log("Task starting:", taskName);
85+
resourceMonitor.startTask(taskName);
86+
currentTaskName = taskName;
87+
},
88+
afterAll: () => {
89+
console.log("Task ending:", currentTaskName);
90+
resourceMonitor.endTask();
91+
currentTaskName = null;
92+
},
93+
},
94+
);
95+
96+
const transformEndpoints = [
97+
{
98+
path: "/api/auth/login",
99+
method: "POST" as const,
100+
body: { email: "user@example.com", password: "super-secret-password-123" },
101+
},
102+
{
103+
path: "/api/users",
104+
method: "POST" as const,
105+
body: {
106+
username: "testuser",
107+
email: "test@example.com",
108+
ssn: "123-45-6789",
109+
creditCard: "4111-1111-1111-1111",
110+
},
111+
},
112+
];
113+
114+
let endpointIndex = 0;
115+
bench.add(
116+
`Transform endpoints (${testName})`,
117+
async () => {
118+
const endpoint = transformEndpoints[endpointIndex % transformEndpoints.length];
119+
endpointIndex++;
120+
121+
const response = await fetch(`${serverUrl}${endpoint.path}`, {
122+
method: endpoint.method,
123+
headers: { "Content-Type": "application/json" },
124+
body: JSON.stringify(endpoint.body),
125+
});
126+
if (!response.ok) {
127+
throw new Error(`HTTP error! status: ${response.status}`);
128+
}
129+
await response.json();
130+
},
131+
{
132+
beforeAll: () => {
133+
const taskName = `Transform endpoints (${testName})`;
134+
console.log("Task starting:", taskName);
135+
resourceMonitor.startTask(taskName);
136+
currentTaskName = taskName;
137+
},
138+
afterAll: () => {
139+
console.log("Task ending:", currentTaskName);
140+
resourceMonitor.endTask();
141+
currentTaskName = null;
142+
},
143+
},
144+
);
145+
146+
resourceMonitor.start();
147+
const runStartedAt = Date.now();
148+
await bench.run();
149+
const benchmarkDurationMs = Date.now() - runStartedAt;
150+
151+
resourceMonitor.stop();
152+
153+
const label = process.env.BENCHMARK_RESULT_LABEL ?? "benchmark";
154+
const benchmarkResult = createBenchmarkRunResult(
155+
bench,
156+
resourceMonitor,
157+
benchmarkDurationMs,
158+
label,
159+
);
160+
const outputPath = persistBenchmarkResult(benchmarkResult);
161+
console.log(`Benchmark results saved to ${outputPath}`);
162+
163+
t.pass();
164+
});
165+
}
166+
167+
export default main;

‎benchmarks/bench/resource-monitor.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
import { TaskResourceStats } from "./result-utils";
2+
3+
interface ResourceMonitorOptions {
4+
intervalMs?: number;
5+
enableMemoryTracking?: boolean;
6+
}
7+
8+
interface TaskStats {
9+
memory: {
10+
rssSum: number;
11+
rssMax: number;
12+
};
13+
startCpuUsage: NodeJS.CpuUsage;
14+
startTime: number;
15+
count: number;
16+
}
17+
18+
interface CompletedTaskStats {
19+
resourceStats: TaskResourceStats;
20+
endTime: number;
21+
}
22+
23+
export class ResourceMonitor {
24+
private taskStats: Map<string, TaskStats> = new Map();
25+
private completedTaskStats: Map<string, CompletedTaskStats> = new Map();
26+
private intervalId: NodeJS.Timeout | null = null;
27+
private currentTaskName: string | null = null;
28+
private currentTaskStats: TaskStats | null = null;
29+
private isRunning: boolean = false;
30+
private options: Required<ResourceMonitorOptions>;
31+
32+
constructor(options: ResourceMonitorOptions = {}) {
33+
this.options = {
34+
intervalMs: options.intervalMs ?? 100,
35+
enableMemoryTracking: options.enableMemoryTracking ?? true,
36+
};
37+
}
38+
39+
start(): void {
40+
this.isRunning = true;
41+
42+
if (this.options.enableMemoryTracking) {
43+
this.intervalId = setInterval(() => {
44+
this.collectMemorySample();
45+
}, this.options.intervalMs);
46+
}
47+
}
48+
49+
stop(): void {
50+
this.isRunning = false;
51+
if (this.intervalId) {
52+
clearInterval(this.intervalId);
53+
this.intervalId = null;
54+
}
55+
}
56+
57+
startTask(taskName: string): void {
58+
this.currentTaskName = taskName;
59+
const startCpuUsage = process.cpuUsage();
60+
const startTime = Date.now();
61+
const taskStats = {
62+
memory: {
63+
rssSum: 0,
64+
rssMax: 0,
65+
},
66+
startCpuUsage,
67+
startTime,
68+
count: 0,
69+
};
70+
this.taskStats.set(taskName, taskStats);
71+
this.currentTaskStats = taskStats;
72+
}
73+
74+
endTask(): void {
75+
if (this.currentTaskName && this.currentTaskStats) {
76+
const endTime = Date.now();
77+
const totalElapsedMs = endTime - this.currentTaskStats.startTime;
78+
const totalElapsedMicroseconds = totalElapsedMs * 1000;
79+
80+
const totalCpuUsage = process.cpuUsage(this.currentTaskStats.startCpuUsage);
81+
82+
const userPercent =
83+
totalElapsedMicroseconds > 0 ? (totalCpuUsage.user / totalElapsedMicroseconds) * 100 : 0;
84+
const systemPercent =
85+
totalElapsedMicroseconds > 0 ? (totalCpuUsage.system / totalElapsedMicroseconds) * 100 : 0;
86+
const totalPercent = userPercent + systemPercent;
87+
88+
const resourceStats: TaskResourceStats = {
89+
cpu: {
90+
userPercent,
91+
systemPercent,
92+
totalPercent,
93+
},
94+
memory: {
95+
rss:
96+
this.currentTaskStats.count > 0
97+
? {
98+
avg: this.currentTaskStats.memory.rssSum / this.currentTaskStats.count,
99+
max: this.currentTaskStats.memory.rssMax,
100+
}
101+
: {
102+
avg: 0,
103+
max: 0,
104+
},
105+
},
106+
};
107+
108+
this.completedTaskStats.set(this.currentTaskName, {
109+
resourceStats,
110+
endTime,
111+
});
112+
}
113+
114+
this.currentTaskName = null;
115+
this.currentTaskStats = null;
116+
}
117+
118+
private collectMemorySample(): void {
119+
if (!this.isRunning || !this.currentTaskStats) return;
120+
121+
const memoryUsage = process.memoryUsage();
122+
this.currentTaskStats.memory.rssSum += memoryUsage.rss;
123+
this.currentTaskStats.memory.rssMax = Math.max(
124+
this.currentTaskStats.memory.rssMax,
125+
memoryUsage.rss,
126+
);
127+
this.currentTaskStats.count++;
128+
}
129+
130+
getTaskStats(taskName: string): TaskResourceStats | null {
131+
const completedStats = this.completedTaskStats.get(taskName);
132+
return completedStats ? completedStats.resourceStats : null;
133+
}
134+
135+
getAllTaskNames(): string[] {
136+
return Array.from(this.completedTaskStats.keys());
137+
}
138+
}

0 commit comments

Comments
 (0)