Skip to content

Commit 9fe8fc0

Browse files
lohanidamodarlohanidamodar
committed
feat(usage): benchmark harness + query_id propagation
1 parent 11fb577 commit 9fe8fc0

9 files changed

Lines changed: 639 additions & 2 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
vendor/
22
.phpunit.result.cache
33
.phpunit.cache/
4+
tests/Benchmark/output/

‎composer.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
"lint": "./vendor/bin/pint --test",
1414
"format": "./vendor/bin/pint",
1515
"check": "./vendor/bin/phpstan analyse --level max src tests",
16-
"test": "./vendor/bin/phpunit --configuration phpunit.xml tests"
16+
"test": "./vendor/bin/phpunit --configuration phpunit.xml --testsuite \"Test Suite\"",
17+
"bench": "./vendor/bin/phpunit --configuration phpunit.xml --testsuite Benchmark"
1718
},
1819
"minimum-stability": "stable",
1920
"require": {

‎phpunit.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
<testsuites>
77
<testsuite name="Test Suite">
88
<directory>tests</directory>
9+
<exclude>tests/Benchmark</exclude>
10+
</testsuite>
11+
<testsuite name="Benchmark">
12+
<directory suffix="Bench.php">tests/Benchmark</directory>
913
</testsuite>
1014
</testsuites>
1115
<coverage processUncoveredFiles="true">

‎src/Usage/Adapter/ClickHouse.php‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ class ClickHouse extends SQL
123123
/** @var bool Whether to wait for async insert confirmation before returning */
124124
private bool $asyncInsertWait = true;
125125

126+
/**
127+
* Opt-in query_id forwarded to ClickHouse on the next query() call only.
128+
* Cleared after a single use so callers must set it explicitly per query.
129+
*/
130+
private ?string $nextQueryId = null;
131+
126132
/**
127133
* @param string $host ClickHouse host
128134
* @param string $username ClickHouse username (default: 'default')
@@ -255,6 +261,20 @@ public function setAsyncInserts(bool $enable, bool $waitForConfirmation = true):
255261
return $this;
256262
}
257263

264+
/**
265+
* Forward an opt-in `query_id` to ClickHouse on the next `query()` call.
266+
*
267+
* Single-use: cleared automatically after one query is dispatched. Pass
268+
* null to clear without dispatching. Benchmarks use this to look up the
269+
* matching row in `system.query_log` (rows_read / read_bytes /
270+
* query_duration_ms).
271+
*/
272+
public function setNextQueryId(?string $queryId): self
273+
{
274+
$this->nextQueryId = $queryId;
275+
return $this;
276+
}
277+
258278
/**
259279
* Get connection statistics for monitoring.
260280
*
@@ -739,11 +759,17 @@ private function buildErrorMessage(string $baseMessage, ?string $table = null, ?
739759
*/
740760
private function query(string $sql, array $params = []): string
741761
{
762+
$queryId = $this->nextQueryId;
763+
$this->nextQueryId = null;
764+
742765
return $this->executeWithRetry(
743-
function (int $attempt) use ($sql, $params): string {
766+
function (int $attempt) use ($sql, $params, $queryId): string {
744767
$startTime = microtime(true);
745768
$scheme = $this->secure ? 'https' : 'http';
746769
$url = "{$scheme}://{$this->host}:{$this->port}/";
770+
if ($queryId !== null && $queryId !== '') {
771+
$url .= '?' . http_build_query(['query_id' => $queryId]);
772+
}
747773

748774
$this->client->addHeader('X-ClickHouse-Database', $this->database);
749775

‎tests/Benchmark/BenchmarkBase.php‎

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,313 @@
1+
<?php
2+
3+
namespace Utopia\Tests\Benchmark;
4+
5+
use PHPUnit\Framework\TestCase;
6+
use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter;
7+
use Utopia\Usage\Usage;
8+
9+
/**
10+
* Base class for ClickHouse benchmarks.
11+
*
12+
* Each scenario seeds a synthetic dataset via `INSERT … SELECT FROM
13+
* numbers(…)`, then runs the target query N times with a fresh query_id and
14+
* records both wall-clock (PHP-side) and ClickHouse-side stats (`rows_read`,
15+
* `read_bytes`, `query_duration_ms` from `system.query_log`).
16+
*
17+
* Output is a JSON report with p50/p95 per scenario, written to
18+
* `tests/Benchmark/output/<class>.json`.
19+
*/
20+
abstract class BenchmarkBase extends TestCase
21+
{
22+
protected Usage $usage;
23+
24+
protected ClickHouseAdapter $adapter;
25+
26+
/** @var array<string, array<int, array{wall_ms: float, rows_read: int, read_bytes: int, query_duration_ms: float}>> */
27+
protected array $results = [];
28+
29+
/** @var int Default number of synthetic rows; override via BENCH_ROWS env */
30+
protected int $defaultRows = 1_000_000;
31+
32+
protected string $tenant = '1';
33+
34+
protected string $namespace = 'utopia_usage_bench';
35+
36+
protected string $metric = 'network.requests';
37+
38+
/** Number of measured iterations (after one warmup). */
39+
protected int $iterations = 5;
40+
41+
protected function setUp(): void
42+
{
43+
$host = getenv('CLICKHOUSE_HOST') ?: 'clickhouse';
44+
$username = getenv('CLICKHOUSE_USER') ?: 'default';
45+
$password = getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse';
46+
$port = (int) (getenv('CLICKHOUSE_PORT') ?: 8123);
47+
$secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false);
48+
49+
$this->adapter = new ClickHouseAdapter($host, $username, $password, $port, $secure);
50+
$this->adapter->setNamespace($this->namespace);
51+
// Benchmarks mirror the cloud workload (sharedTables=true) so the
52+
// synthetic seed and the routing paths exercise the same schema.
53+
$this->adapter->setSharedTables(true);
54+
$this->adapter->setTenant($this->tenant);
55+
56+
if ($database = getenv('CLICKHOUSE_DATABASE')) {
57+
$this->adapter->setDatabase($database);
58+
}
59+
60+
$this->usage = new Usage($this->adapter);
61+
$this->usage->setup();
62+
63+
$rowsEnv = getenv('BENCH_ROWS');
64+
if (is_string($rowsEnv) && ctype_digit($rowsEnv)) {
65+
$this->defaultRows = (int) $rowsEnv;
66+
}
67+
}
68+
69+
protected function tearDown(): void
70+
{
71+
$this->writeReport();
72+
}
73+
74+
/**
75+
* Seed `$n` synthetic rows into the events table via
76+
* `INSERT … SELECT FROM numbers(n)`. Bypasses the library's batching path.
77+
*
78+
* @param int $rows Number of rows to insert
79+
* @param string|null $metric Metric name (defaults to $this->metric)
80+
*/
81+
protected function seedEventRows(int $rows, ?string $metric = null): void
82+
{
83+
$metric ??= $this->metric;
84+
85+
$reflection = new \ReflectionClass($this->adapter);
86+
87+
$getEvents = $reflection->getMethod('getEventsTableName');
88+
$getEvents->setAccessible(true);
89+
$eventsTableRaw = $getEvents->invoke($this->adapter);
90+
$eventsTable = is_string($eventsTableRaw) ? $eventsTableRaw : '';
91+
92+
$getDatabase = $reflection->getProperty('database');
93+
$getDatabase->setAccessible(true);
94+
$databaseRaw = $getDatabase->getValue($this->adapter);
95+
$database = is_string($databaseRaw) ? $databaseRaw : '';
96+
97+
$tableRef = "`{$database}`.`{$eventsTable}`";
98+
99+
$template = file_get_contents(__DIR__ . '/fixtures/seed.sql');
100+
if (!is_string($template)) {
101+
throw new \RuntimeException('Unable to read seed.sql fixture');
102+
}
103+
104+
$sql = strtr($template, [
105+
'{TABLE}' => $tableRef,
106+
'{ROWS}' => (string) $rows,
107+
'{METRIC}' => addslashes($metric),
108+
'{TENANT}' => addslashes($this->tenant),
109+
]);
110+
111+
$this->runRawSql($sql);
112+
}
113+
114+
/**
115+
* Seed `$n` synthetic rows into the gauges table.
116+
*/
117+
protected function seedGaugeRows(int $rows, string $metric = 'storage'): void
118+
{
119+
$reflection = new \ReflectionClass($this->adapter);
120+
121+
$getGauges = $reflection->getMethod('getGaugesTableName');
122+
$getGauges->setAccessible(true);
123+
$gaugesTableRaw = $getGauges->invoke($this->adapter);
124+
$gaugesTable = is_string($gaugesTableRaw) ? $gaugesTableRaw : '';
125+
126+
$getDatabase = $reflection->getProperty('database');
127+
$getDatabase->setAccessible(true);
128+
$databaseRaw = $getDatabase->getValue($this->adapter);
129+
$database = is_string($databaseRaw) ? $databaseRaw : '';
130+
131+
$tableRef = "`{$database}`.`{$gaugesTable}`";
132+
133+
$sql = "INSERT INTO {$tableRef} (id, metric, value, time, tenant) "
134+
. "SELECT lower(hex(randomString(16))), '" . addslashes($metric) . "', "
135+
. "number AS value, now() - toIntervalSecond(number % 86400) AS time, "
136+
. "'" . addslashes($this->tenant) . "' AS tenant "
137+
. "FROM numbers({$rows})";
138+
139+
$this->runRawSql($sql);
140+
}
141+
142+
/**
143+
* Execute a raw SQL string against ClickHouse (bypassing parameter binding).
144+
*/
145+
protected function runRawSql(string $sql): void
146+
{
147+
$reflection = new \ReflectionClass($this->adapter);
148+
$query = $reflection->getMethod('query');
149+
$query->setAccessible(true);
150+
$query->invoke($this->adapter, $sql, []);
151+
}
152+
153+
/**
154+
* Truncate the events / gauges tables and recreate the rollup tables.
155+
*/
156+
protected function purgeAll(): void
157+
{
158+
$this->usage->purge();
159+
}
160+
161+
/**
162+
* Run a benchmark scenario `$iterations + 1` times (1 warmup) and record
163+
* per-iteration stats into $this->results[$name].
164+
*
165+
* @param string $name Scenario name
166+
* @param callable(string): void $callable Receives a unique query_id; must
167+
* forward it to the adapter via
168+
* `setNextQueryId()` before its
169+
* ClickHouse call.
170+
*/
171+
protected function runBench(string $name, callable $callable, ?int $iterations = null): void
172+
{
173+
$iterations ??= $this->iterations;
174+
175+
// Warmup pass — discarded to dodge cold-start variance.
176+
$warmupId = $this->generateQueryId($name, -1);
177+
$callable($warmupId);
178+
179+
$records = [];
180+
for ($i = 0; $i < $iterations; $i++) {
181+
$queryId = $this->generateQueryId($name, $i);
182+
183+
$start = microtime(true);
184+
$callable($queryId);
185+
$wallMs = (microtime(true) - $start) * 1000.0;
186+
187+
$chStats = $this->captureCHStats($queryId);
188+
$chStats['wall_ms'] = round($wallMs, 3);
189+
190+
$records[] = $chStats;
191+
}
192+
193+
$this->results[$name] = $records;
194+
}
195+
196+
/**
197+
* Query system.query_log for the given query_id and return rows_read,
198+
* read_bytes, query_duration_ms (best-effort — returns zeros if the log
199+
* hasn't flushed yet).
200+
*
201+
* @return array{wall_ms: float, rows_read: int, read_bytes: int, query_duration_ms: float}
202+
*/
203+
protected function captureCHStats(string $queryId): array
204+
{
205+
$stats = [
206+
'wall_ms' => 0.0,
207+
'rows_read' => 0,
208+
'read_bytes' => 0,
209+
'query_duration_ms' => 0.0,
210+
];
211+
212+
// system.query_log is buffered — flush before we read it.
213+
try {
214+
$this->runRawSql('SYSTEM FLUSH LOGS');
215+
} catch (\Throwable $e) {
216+
// Some environments deny SYSTEM access; fall through with zeros.
217+
return $stats;
218+
}
219+
220+
$escapedId = addslashes($queryId);
221+
$sql = "SELECT sum(read_rows) AS rows_read, sum(read_bytes) AS read_bytes, "
222+
. "max(query_duration_ms) AS query_duration_ms "
223+
. "FROM system.query_log "
224+
. "WHERE query_id = '{$escapedId}' AND type >= 2 "
225+
. "FORMAT JSON";
226+
227+
try {
228+
$reflection = new \ReflectionClass($this->adapter);
229+
$query = $reflection->getMethod('query');
230+
$query->setAccessible(true);
231+
$raw = $query->invoke($this->adapter, $sql, []);
232+
$rawString = is_string($raw) ? $raw : '';
233+
$json = json_decode($rawString, true);
234+
if (is_array($json) && isset($json['data'][0]) && is_array($json['data'][0])) {
235+
$row = $json['data'][0];
236+
$stats['rows_read'] = (int) ($row['rows_read'] ?? 0);
237+
$stats['read_bytes'] = (int) ($row['read_bytes'] ?? 0);
238+
$stats['query_duration_ms'] = (float) ($row['query_duration_ms'] ?? 0);
239+
}
240+
} catch (\Throwable $e) {
241+
// Best-effort.
242+
}
243+
244+
return $stats;
245+
}
246+
247+
/**
248+
* Compute percentile for a sorted numeric series.
249+
*
250+
* @param array<int|float> $values
251+
*/
252+
private function percentile(array $values, float $p): float
253+
{
254+
if (empty($values)) {
255+
return 0.0;
256+
}
257+
sort($values);
258+
$idx = (int) floor($p * (count($values) - 1));
259+
return (float) $values[$idx];
260+
}
261+
262+
/**
263+
* Write the per-scenario JSON report under tests/Benchmark/output/.
264+
*/
265+
protected function writeReport(): void
266+
{
267+
if (empty($this->results)) {
268+
return;
269+
}
270+
271+
$summary = [];
272+
foreach ($this->results as $name => $records) {
273+
$wall = array_map(static fn (array $r) => $r['wall_ms'], $records);
274+
$rows = array_map(static fn (array $r) => $r['rows_read'], $records);
275+
$bytes = array_map(static fn (array $r) => $r['read_bytes'], $records);
276+
$chDur = array_map(static fn (array $r) => $r['query_duration_ms'], $records);
277+
278+
$summary[$name] = [
279+
'iterations' => count($records),
280+
'rows_dataset' => $this->defaultRows,
281+
'wall_p50_ms' => $this->percentile($wall, 0.50),
282+
'wall_p95_ms' => $this->percentile($wall, 0.95),
283+
'ch_p50_ms' => $this->percentile($chDur, 0.50),
284+
'ch_p95_ms' => $this->percentile($chDur, 0.95),
285+
'rows_read_p50' => $this->percentile($rows, 0.50),
286+
'rows_read_p95' => $this->percentile($rows, 0.95),
287+
'read_bytes_p50' => $this->percentile($bytes, 0.50),
288+
'read_bytes_p95' => $this->percentile($bytes, 0.95),
289+
'samples' => $records,
290+
];
291+
}
292+
293+
$outDir = __DIR__ . '/output';
294+
if (!is_dir($outDir)) {
295+
@mkdir($outDir, 0775, true);
296+
}
297+
298+
$short = (new \ReflectionClass($this))->getShortName();
299+
$file = $outDir . '/' . $short . '.json';
300+
file_put_contents($file, (string) json_encode([
301+
'class' => static::class,
302+
'tenant' => $this->tenant,
303+
'rows_dataset' => $this->defaultRows,
304+
'generated_at' => date(DATE_ATOM),
305+
'scenarios' => $summary,
306+
], JSON_PRETTY_PRINT));
307+
}
308+
309+
protected function generateQueryId(string $scenario, int $iter): string
310+
{
311+
return 'bench-' . bin2hex(random_bytes(8)) . '-' . preg_replace('/[^a-z0-9_]+/i', '_', $scenario) . '-' . $iter;
312+
}
313+
}

0 commit comments

Comments
 (0)