Skip to content

Commit 1c64321

Browse files
committed
Move benchmark make target
1 parent 2a01c00 commit 1c64321

4 files changed

Lines changed: 94 additions & 42 deletions

File tree

Makefile

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
SHELL := /bin/bash
2-
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin test-guestmemory-linux test-guestmemory-vz install-tools gen-jwt download-ch-binaries download-firecracker-binaries download-ch-spec ensure-ch-binaries ensure-firecracker-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded bench-activity-ramp
2+
.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin test-guestmemory-linux test-guestmemory-vz install-tools gen-jwt download-ch-binaries download-firecracker-binaries download-ch-spec ensure-ch-binaries ensure-firecracker-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded
33

44
# Directory where local binaries will be installed
55
BIN_DIR ?= $(CURDIR)/bin
@@ -15,17 +15,6 @@ AIR ?= $(BIN_DIR)/air
1515
WIRE ?= $(BIN_DIR)/wire
1616
XCADDY ?= $(BIN_DIR)/xcaddy
1717
TEST_TIMEOUT ?= $(GO_TEST_TIMEOUT)
18-
K6 ?= k6
19-
K6_OUT_DIR ?= .bench/k6
20-
HYPEMAN_BASE_URL ?= http://127.0.0.1:8080
21-
HYPEMAN_IMAGE ?= docker.io/library/nginx:alpine
22-
HYPEMAN_BENCH_MAX_VUS ?= 16
23-
HYPEMAN_BENCH_VU_STEP ?= 1
24-
HYPEMAN_BENCH_STAGE_DURATION ?= 2m
25-
HYPEMAN_BENCH_DASHBOARD_PERIOD ?= 120s
26-
HYPEMAN_HYPERVISOR ?= cloud-hypervisor
27-
HYPEMAN_INGRESS_HOST_PORT ?= 80
28-
HYPEMAN_CREATE_REJECTED_BACKOFF_SECONDS ?= 1
2918

3019
# Install oapi-codegen (pinned to match committed generated code)
3120
$(OAPI_CODEGEN): | $(BIN_DIR)
@@ -300,35 +289,6 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries
300289
go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
301290
fi
302291

303-
bench-activity-ramp:
304-
@if ! command -v $(K6) >/dev/null 2>&1; then \
305-
echo "k6 not found; install k6 or run with K6=/path/to/k6"; \
306-
exit 1; \
307-
fi
308-
@if [ -z "$$HYPEMAN_API_KEY" ]; then \
309-
echo "HYPEMAN_API_KEY is required"; \
310-
exit 1; \
311-
fi
312-
@mkdir -p $(K6_OUT_DIR)
313-
K6_WEB_DASHBOARD=true \
314-
K6_WEB_DASHBOARD_PORT=-1 \
315-
K6_WEB_DASHBOARD_PERIOD=$(HYPEMAN_BENCH_DASHBOARD_PERIOD) \
316-
K6_WEB_DASHBOARD_EXPORT=$(K6_OUT_DIR)/activity-ramp.html \
317-
$(K6) run \
318-
--summary-mode=full \
319-
--summary-trend-stats="avg,med,p(90),p(95),p(99),min,max" \
320-
--summary-export=$(K6_OUT_DIR)/activity-ramp-summary.json \
321-
-e HYPEMAN_BASE_URL="$(HYPEMAN_BASE_URL)" \
322-
-e HYPEMAN_API_KEY="$$HYPEMAN_API_KEY" \
323-
-e HYPEMAN_IMAGE="$(HYPEMAN_IMAGE)" \
324-
-e HYPEMAN_HYPERVISOR="$(HYPEMAN_HYPERVISOR)" \
325-
-e HYPEMAN_BENCH_MAX_VUS="$(HYPEMAN_BENCH_MAX_VUS)" \
326-
-e HYPEMAN_BENCH_VU_STEP="$(HYPEMAN_BENCH_VU_STEP)" \
327-
-e HYPEMAN_BENCH_STAGE_DURATION="$(HYPEMAN_BENCH_STAGE_DURATION)" \
328-
-e HYPEMAN_INGRESS_HOST_PORT="$(HYPEMAN_INGRESS_HOST_PORT)" \
329-
-e HYPEMAN_CREATE_REJECTED_BACKOFF_SECONDS="$(HYPEMAN_CREATE_REJECTED_BACKOFF_SECONDS)" \
330-
benchmarks/k6/activity-ramp.ts
331-
332292
# macOS tests (no sudo needed, adds e2fsprogs to PATH)
333293
# Uses 'go list' to discover compilable packages, then filters out packages
334294
# whose test files reference Linux-only symbols (network, devices, system/init).

benchmarks/Makefile

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
SHELL := /bin/bash
2+
3+
.PHONY: bench-activity-ramp
4+
5+
REPO_ROOT := $(abspath $(CURDIR)/..)
6+
7+
K6 ?= k6
8+
K6_OUT_DIR ?= $(REPO_ROOT)/.bench/k6
9+
HYPEMAN_BASE_URL ?= http://127.0.0.1:8080
10+
HYPEMAN_IMAGE ?= docker.io/library/nginx:alpine
11+
HYPEMAN_BENCH_MAX_VUS ?= 16
12+
HYPEMAN_BENCH_VU_STEP ?= 1
13+
HYPEMAN_BENCH_STAGE_DURATION ?= 2m
14+
HYPEMAN_BENCH_DASHBOARD_PERIOD ?= 120s
15+
HYPEMAN_HYPERVISOR ?= cloud-hypervisor
16+
HYPEMAN_INGRESS_HOST_PORT ?= 80
17+
HYPEMAN_CREATE_REJECTED_BACKOFF_SECONDS ?= 1
18+
19+
bench-activity-ramp:
20+
@if ! command -v $(K6) >/dev/null 2>&1; then \
21+
echo "k6 not found; install k6 or run with K6=/path/to/k6"; \
22+
exit 1; \
23+
fi
24+
@if [ -z "$$HYPEMAN_API_KEY" ]; then \
25+
echo "HYPEMAN_API_KEY is required"; \
26+
exit 1; \
27+
fi
28+
@mkdir -p $(K6_OUT_DIR)
29+
K6_WEB_DASHBOARD=true \
30+
K6_WEB_DASHBOARD_PORT=-1 \
31+
K6_WEB_DASHBOARD_PERIOD=$(HYPEMAN_BENCH_DASHBOARD_PERIOD) \
32+
K6_WEB_DASHBOARD_EXPORT=$(K6_OUT_DIR)/activity-ramp.html \
33+
$(K6) run \
34+
--summary-mode=full \
35+
--summary-trend-stats="avg,med,p(90),p(95),p(99),min,max" \
36+
--summary-export=$(K6_OUT_DIR)/activity-ramp-summary.json \
37+
-e HYPEMAN_BASE_URL="$(HYPEMAN_BASE_URL)" \
38+
-e HYPEMAN_API_KEY="$$HYPEMAN_API_KEY" \
39+
-e HYPEMAN_IMAGE="$(HYPEMAN_IMAGE)" \
40+
-e HYPEMAN_HYPERVISOR="$(HYPEMAN_HYPERVISOR)" \
41+
-e HYPEMAN_BENCH_MAX_VUS="$(HYPEMAN_BENCH_MAX_VUS)" \
42+
-e HYPEMAN_BENCH_VU_STEP="$(HYPEMAN_BENCH_VU_STEP)" \
43+
-e HYPEMAN_BENCH_STAGE_DURATION="$(HYPEMAN_BENCH_STAGE_DURATION)" \
44+
-e HYPEMAN_INGRESS_HOST_PORT="$(HYPEMAN_INGRESS_HOST_PORT)" \
45+
-e HYPEMAN_CREATE_REJECTED_BACKOFF_SECONDS="$(HYPEMAN_CREATE_REJECTED_BACKOFF_SECONDS)" \
46+
k6/activity-ramp.ts

benchmarks/k6/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ The default ramp increases concurrency by one virtual user every two minutes up
1515

1616
```sh
1717
export HYPEMAN_API_KEY=...
18-
make bench-activity-ramp \
18+
make -C benchmarks bench-activity-ramp \
1919
HYPEMAN_BASE_URL=http://127.0.0.1:8080 \
2020
HYPEMAN_IMAGE=docker.io/library/nginx:alpine \
2121
HYPEMAN_HYPERVISOR=cloud-hypervisor \

benchmarks/k6/activity-ramp.ts

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,16 @@ import { Counter, Rate, Trend } from 'k6/metrics';
55

66
type Tags = Record<string, string>;
77

8+
// k6 runs this file in a few phases:
9+
//
10+
// 1. The module top level runs once per virtual user (VU). Put metrics,
11+
// options, and helper definitions here.
12+
// 2. setup() runs once before load starts. We use it to verify the Hypeman API,
13+
// ensure the image exists, and create the shared pattern ingress.
14+
// 3. The default function is the workload. k6 calls it repeatedly in every VU
15+
// while the ramping-vus scenario is active.
16+
// 4. teardown() runs once after load stops. It removes any instances tagged
17+
// with this benchmark run ID.
818
interface Config {
919
baseUrl: string;
1020
apiKey: string;
@@ -33,12 +43,17 @@ interface Config {
3343
imageReadyTimeoutSeconds: number;
3444
}
3545

46+
// Trend metrics store latency distributions. Passing true tells k6 these values
47+
// are durations in milliseconds, so summaries and dashboards format them as time.
3648
const createMs = new Trend('hypeman_create_instance_ms', true);
3749
const waitRunningMs = new Trend('hypeman_wait_running_ms', true);
3850
const probeReadyMs = new Trend('hypeman_probe_ready_ms', true);
3951
const probeHTTPMs = new Trend('hypeman_probe_http_ms', true);
4052
const deleteMs = new Trend('hypeman_delete_instance_ms', true);
4153
const activityMs = new Trend('hypeman_activity_total_ms', true);
54+
55+
// Rate metrics track the fraction of samples that were true. Counter metrics
56+
// track raw counts. These give us capacity signals alongside latency.
4257
const activityOk = new Rate('hypeman_activity_ok');
4358
const cleanupOk = new Rate('hypeman_cleanup_ok');
4459
const createRejected = new Rate('hypeman_create_rejected');
@@ -51,6 +66,8 @@ export const options = {
5166
setupTimeout: '15m',
5267
teardownTimeout: '10m',
5368
scenarios: {
69+
// ramping-vus changes the number of concurrent virtual users over time.
70+
// Each active VU loops through the activity until k6 lowers concurrency.
5471
activity_ramp: {
5572
executor: 'ramping-vus',
5673
startVUs: config.startVUs,
@@ -59,12 +76,17 @@ export const options = {
5976
},
6077
},
6178
thresholds: {
79+
// Thresholds mark the run failed if cleanup or probe success gets too low.
80+
// Create rejections are measured separately because they are the capacity
81+
// signal we are trying to find, not a script bug by themselves.
6282
hypeman_cleanup_ok: ['rate>0.95'],
6383
hypeman_probe_ok: ['rate>0.80'],
6484
},
6585
};
6686

6787
export function setup() {
88+
// setup() returns data that k6 passes into every default() iteration.
89+
// The run ID is shared so all VUs use the same cleanup tag.
6890
checkRequiredConfig(config);
6991
ensureHealthy();
7092
ensureImageReady(config.image);
@@ -76,6 +98,9 @@ export function setup() {
7698
}
7799

78100
export default function (data: { runId: string }) {
101+
// One iteration is one full user-facing activity:
102+
// create -> wait for Running -> send one HTTP probe -> delete.
103+
// k6 repeats this loop in each VU for as long as that VU is scheduled.
79104
const iterationStart = Date.now();
80105
const instanceName = instanceNameFor(data.runId);
81106
const tags: Tags = {
@@ -90,6 +115,8 @@ export default function (data: { runId: string }) {
90115
try {
91116
created = createInstance(instanceName, tags);
92117
if (!created) {
118+
// A false return means Hypeman rejected the create due to capacity. The
119+
// rejection was already counted, so this VU ends the iteration quietly.
93120
return;
94121
}
95122
waitForRunning(instanceName, tags);
@@ -106,6 +133,7 @@ export default function (data: { runId: string }) {
106133
}
107134

108135
export function teardown(data: { runId: string }) {
136+
// Best-effort cleanup handles interrupted iterations or a failed test run.
109137
cleanupRunInstances(data.runId);
110138
}
111139

@@ -143,6 +171,8 @@ function loadConfig(): Config {
143171
}
144172

145173
function rampStages(cfg: Config): Array<{ duration: string; target: number }> {
174+
// Stages are the k6 ramp plan. With the defaults this produces:
175+
// 1 VU start, then 2, 3, 4, ... 16 VUs, spending 2 minutes at each target.
146176
const stages: Array<{ duration: string; target: number }> = [];
147177
for (let target = cfg.startVUs + cfg.vuStep; target <= cfg.maxVUs; target += cfg.vuStep) {
148178
stages.push({ duration: cfg.stageDuration, target });
@@ -172,6 +202,8 @@ function ensureHealthy() {
172202
}
173203

174204
function ensureImageReady(image: string) {
205+
// Hypeman imports images asynchronously. The benchmark should measure
206+
// instance lifecycle under load, not image import time, so setup waits here.
175207
let imageBody = findImage(image);
176208
if (!imageBody) {
177209
const res = apiPost('/images', { name: image }, { kind: 'setup', step: 'image-create' });
@@ -211,6 +243,9 @@ function findImage(image: string): { status?: string; error?: string } | null {
211243
}
212244

213245
function ensurePatternIngress() {
246+
// The ingress uses a hostname pattern where {instance} is replaced by each
247+
// instance name. That lets all iterations share one ingress instead of
248+
// creating and deleting ingress resources inside the hot loop.
214249
const encoded = encodeURIComponent(config.ingressName);
215250
const existing = apiGet(`/ingresses/${encoded}`, { kind: 'setup', step: 'ingress-get' });
216251
if (existing.status === 200) {
@@ -281,6 +316,8 @@ function createInstance(name: string, tags: Tags): boolean {
281316
return true;
282317
}
283318
if (res.status === 409) {
319+
// 409 is useful data: it means the server admitted that this concurrency
320+
// level is beyond current capacity. Count it without failing the script.
284321
createRejected.add(true, tags);
285322
createRejections.add(1, tags);
286323
sleep(config.createRejectedBackoffSeconds);
@@ -291,6 +328,7 @@ function createInstance(name: string, tags: Tags): boolean {
291328
}
292329

293330
function waitForRunning(name: string, tags: Tags) {
331+
// This measures control-plane latency from accepted create to Running state.
294332
const started = Date.now();
295333
const path = `/instances/${encodeURIComponent(name)}`;
296334
const deadline = started + config.waitTimeoutSeconds * 1000;
@@ -315,6 +353,9 @@ function waitForRunning(name: string, tags: Tags) {
315353
}
316354

317355
function probeInstance(name: string, tags: Tags) {
356+
// The probe goes through the shared ingress URL. The Host header selects the
357+
// instance via the pattern ingress, so latency here reflects the data path
358+
// through Hypeman into the guest workload.
318359
const started = Date.now();
319360
const probeURL = `${config.probeUrl}${config.probePath.startsWith('/') ? config.probePath : `/${config.probePath}`}`;
320361
const host = `${name}${config.probeHostSuffix}`;
@@ -350,6 +391,7 @@ function deleteInstance(name: string, tags: Tags): boolean {
350391
}
351392

352393
function cleanupRunInstances(runId: string) {
394+
// Query by benchmark tags so teardown only touches instances from this run.
353395
const query = `tags%5Bbenchmark%5D=activity-ramp&tags%5Brun_id%5D=${encodeURIComponent(runId)}`;
354396
const res = apiGet(`/instances?${query}`, { kind: 'teardown', step: 'list-run-instances', run_id: runId });
355397
if (res.status !== 200) {
@@ -417,10 +459,14 @@ function assertStatus(res: RefinedResponse<ResponseType | undefined>, allowed: n
417459
}
418460

419461
function tagStep(tags: Tags, step: string): Tags {
462+
// Tags are attached to k6 metric samples. They make it possible to filter
463+
// results by step, hypervisor, run ID, or instance in JSON outputs.
420464
return { ...tags, step };
421465
}
422466

423467
function instanceNameFor(runId: string): string {
468+
// k6 exposes the current virtual user and iteration through k6/execution.
469+
// Including both values keeps names unique even when many VUs run at once.
424470
const vu = exec.vu.idInTest;
425471
const iter = exec.scenario.iterationInTest;
426472
const suffix = `-${vu}-${iter}`;

0 commit comments

Comments
 (0)