Skip to content

Commit 246c79a

Browse files
lohanidamodarlohanidamodar
committed
feat(usage): Plausible-style schema hygiene (codecs, LowCardinality, drop weak blooms, drop FINAL)
1 parent c40f2cd commit 246c79a

3 files changed

Lines changed: 182 additions & 25 deletions

File tree

‎src/Usage/Adapter/ClickHouse.php‎

Lines changed: 61 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1081,14 +1081,14 @@ public function setup(): void
10811081
*/
10821082
private function createTable(string $tableName, string $type, array $indexes): void
10831083
{
1084-
$columns = ['id String'];
1084+
$columns = ['id String ' . $this->getColumnCodec('id')];
10851085

10861086
foreach ($this->getAttributes($type) as $attribute) {
10871087
/** @var string $id */
10881088
$id = $attribute['$id'];
10891089

10901090
if ($id === 'time') {
1091-
$columns[] = 'time DateTime64(3)';
1091+
$columns[] = 'time DateTime64(3) ' . $this->getColumnCodec('time');
10921092
} else {
10931093
$columns[] = $this->getColumnDefinition($id, $type);
10941094
}
@@ -1099,17 +1099,20 @@ private function createTable(string $tableName, string $type, array $indexes): v
10991099
$columns[] = 'tenant Nullable(String)';
11001100
}
11011101

1102-
// Build indexes
1102+
// Build indexes. `indexType` selects per-column between bloom_filter
1103+
// (high-cardinality strings) and set(0) (low-cardinality enums that
1104+
// are too dense for blooms to ever skip — see plausible-research.md).
11031105
$indexDefs = [];
11041106
foreach ($indexes as $index) {
11051107
/** @var string $indexName */
11061108
$indexName = $index['$id'];
11071109
/** @var array<string> $attributes */
11081110
$attributes = $index['attributes'];
1111+
$indexType = is_string($index['indexType'] ?? null) ? $index['indexType'] : 'bloom_filter';
11091112
$escapedIndexName = $this->escapeIdentifier($indexName);
11101113
$escapedAttributes = array_map(fn ($attr) => $this->escapeIdentifier($attr), $attributes);
11111114
$attributeList = implode(', ', $escapedAttributes);
1112-
$indexDefs[] = "INDEX {$escapedIndexName} ({$attributeList}) TYPE bloom_filter GRANULARITY 1";
1115+
$indexDefs[] = "INDEX {$escapedIndexName} ({$attributeList}) TYPE {$indexType} GRANULARITY 1";
11131116
}
11141117

11151118
$escapedDatabaseAndTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($tableName);
@@ -1153,12 +1156,12 @@ private function createDailyTable(): void
11531156
$columns = [
11541157
'metric String',
11551158
'value Int64',
1156-
'time DateTime64(3)',
1159+
'time DateTime64(3) CODEC(Delta(4), LZ4)',
11571160
'resource LowCardinality(Nullable(String))',
1158-
'resourceId Nullable(String)',
1159-
'resourceInternalId Nullable(String)',
1160-
'teamId Nullable(String)',
1161-
'teamInternalId Nullable(String)',
1161+
'resourceId Nullable(String) CODEC(ZSTD(3))',
1162+
'resourceInternalId Nullable(String) CODEC(ZSTD(3))',
1163+
'teamId Nullable(String) CODEC(ZSTD(3))',
1164+
'teamInternalId Nullable(String) CODEC(ZSTD(3))',
11621165
];
11631166

11641167
if ($this->sharedTables) {
@@ -1365,8 +1368,11 @@ private function getColumnType(string $id, string $type = 'event'): string
13651368

13661369
$lowCardinality = [
13671370
'country', 'region', 'service', 'resource',
1368-
'osCode', 'osName', 'clientType', 'clientCode', 'clientName',
1369-
'clientEngine', 'deviceName', 'deviceBrand',
1371+
'osCode', 'osName', 'osVersion',
1372+
'clientType', 'clientCode', 'clientName', 'clientVersion',
1373+
'clientEngine', 'clientEngineVersion',
1374+
'deviceName', 'deviceBrand', 'deviceModel',
1375+
'path', 'hostname',
13701376
];
13711377

13721378
if (in_array($id, $lowCardinality, true)) {
@@ -1389,9 +1395,38 @@ protected function getColumnDefinition(string $id, string $type = 'event'): stri
13891395
{
13901396
$chType = $this->getColumnType($id, $type);
13911397
$escapedId = $this->escapeIdentifier($id);
1398+
$codec = $this->getColumnCodec($id);
1399+
if ($codec !== '') {
1400+
return "{$escapedId} {$chType} {$codec}";
1401+
}
13921402
return "{$escapedId} {$chType}";
13931403
}
13941404

1405+
/**
1406+
* Return the per-column ClickHouse CODEC clause (Plausible-style
1407+
* compression hygiene) for the events/gauges tables. Empty string when
1408+
* no codec is overridden for this column.
1409+
*/
1410+
private function getColumnCodec(string $id): string
1411+
{
1412+
if ($id === 'time') {
1413+
return 'CODEC(Delta(4), LZ4)';
1414+
}
1415+
1416+
$zstdColumns = [
1417+
'id', 'path', 'hostname',
1418+
'resourceId', 'resourceInternalId',
1419+
'teamId', 'teamInternalId',
1420+
'osVersion', 'clientVersion', 'clientEngineVersion', 'deviceModel',
1421+
];
1422+
1423+
if (in_array($id, $zstdColumns, true)) {
1424+
return 'CODEC(ZSTD(3))';
1425+
}
1426+
1427+
return '';
1428+
}
1429+
13951430
/**
13961431
* Validate metric data for batch operations.
13971432
*
@@ -2274,20 +2309,28 @@ public function findDaily(array $queries = []): array
22742309
$parsed = $this->parseQueries($queries, Usage::TYPE_EVENT);
22752310
$whereData = $this->buildWhereClause($parsed['filters'], $parsed['params']);
22762311

2277-
$dailyColumns = ['metric', 'value', 'time'];
2312+
$groupByColumns = ['metric', 'time'];
22782313
if ($this->sharedTables) {
2279-
$dailyColumns[] = 'tenant';
2314+
$groupByColumns[] = 'tenant';
2315+
}
2316+
2317+
$selectExpressions = [];
2318+
foreach ($groupByColumns as $column) {
2319+
$selectExpressions[] = $this->escapeIdentifier($column);
22802320
}
2281-
$selectColumns = implode(', ', array_map(fn ($c) => $this->escapeIdentifier($c), $dailyColumns));
2321+
$selectExpressions[] = 'sum(`value`) AS `value`';
2322+
2323+
$selectColumns = implode(', ', $selectExpressions);
2324+
$groupBySql = implode(', ', array_map(fn ($c) => $this->escapeIdentifier($c), $groupByColumns));
22822325

22832326
$orderClause = !empty($parsed['orderBy']) ? ' ORDER BY ' . implode(', ', $parsed['orderBy']) : '';
22842327
$limitClause = isset($parsed['limit']) ? ' LIMIT {limit:UInt64}' : '';
22852328
$offsetClause = isset($parsed['offset']) ? ' OFFSET {offset:UInt64}' : '';
22862329

2287-
// The daily table is SummingMergeTree. Reading raw rows returns
2288-
// un-merged duplicates until background merges run. FINAL forces
2289-
// merge-on-read so callers always see fully-collapsed values.
2290-
$sql = "SELECT {$selectColumns} FROM {$fromTable} FINAL{$whereData['clause']}{$orderClause}{$limitClause}{$offsetClause} FORMAT JSON";
2330+
// SummingMergeTree collapses on background merges; reading raw rows
2331+
// sums them via explicit GROUP BY at query time. Net effect matches
2332+
// FROM … FINAL but keeps the read parallelisable.
2333+
$sql = "SELECT {$selectColumns} FROM {$fromTable}{$whereData['clause']} GROUP BY {$groupBySql}{$orderClause}{$limitClause}{$offsetClause} FORMAT JSON";
22912334

22922335
return $this->parseResults($this->query($sql, $whereData['params']), Usage::TYPE_EVENT);
22932336
}

‎src/Usage/Metric.php‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -709,9 +709,6 @@ public static function getSchema(): array
709709
*/
710710
public static function getEventIndexes(): array
711711
{
712-
// `metric` and `time` are part of the ClickHouse primary key
713-
// (ORDER BY (tenant, metric, time, id)), so separate bloom_filter
714-
// indexes on them would be redundant.
715712
$indexed = [
716713
'path', 'method', 'status',
717714
'service', 'resource', 'resourceId', 'resourceInternalId',
@@ -720,16 +717,16 @@ public static function getEventIndexes(): array
720717
'osName', 'clientType', 'clientName', 'deviceName',
721718
];
722719

720+
$setIndexed = ['status', 'method', 'country', 'service', 'clientType', 'osName'];
721+
723722
return array_map(
724-
static function (string $col): array {
723+
static function (string $col) use ($setIndexed): array {
725724
$entry = [
726725
'$id' => 'index-' . $col,
727726
'type' => 'key',
728727
'attributes' => [$col],
728+
'indexType' => in_array($col, $setIndexed, true) ? 'set(0)' : 'bloom_filter',
729729
];
730-
// path is sized 1024 for data fidelity; cap the index key at
731-
// 255 so the MariaDB single-attribute index stays within the
732-
// 768-byte InnoDB key prefix limit.
733730
if ($col === 'path') {
734731
$entry['lengths'] = [255];
735732
}
@@ -751,6 +748,7 @@ public static function getGaugeIndexes(): array
751748
'$id' => 'index-' . $col,
752749
'type' => 'key',
753750
'attributes' => [$col],
751+
'indexType' => 'bloom_filter',
754752
],
755753
['resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'],
756754
);

‎tests/Usage/Adapter/ClickHouseSchemaTest.php‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
<?php
2+
3+
namespace Utopia\Tests\Adapter;
4+
5+
use PHPUnit\Framework\TestCase;
6+
use ReflectionClass;
7+
use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter;
8+
use Utopia\Usage\Usage;
9+
10+
/**
11+
* Verifies the Plausible-style schema hygiene applied in P2:
12+
* - explicit CODEC(Delta(4), LZ4) on time
13+
* - CODEC(ZSTD(3)) on high-cardinality string columns
14+
* - LowCardinality on path / hostname / version+model columns
15+
* - set(0) skipping index on low-cardinality enums
16+
*/
17+
class ClickHouseSchemaTest extends TestCase
18+
{
19+
private ClickHouseAdapter $adapter;
20+
21+
protected function setUp(): void
22+
{
23+
$host = getenv('CLICKHOUSE_HOST') ?: 'clickhouse';
24+
$username = getenv('CLICKHOUSE_USER') ?: 'default';
25+
$password = getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse';
26+
$port = (int) (getenv('CLICKHOUSE_PORT') ?: 8123);
27+
$secure = (bool) (getenv('CLICKHOUSE_SECURE') ?: false);
28+
29+
$this->adapter = new ClickHouseAdapter($host, $username, $password, $port, $secure);
30+
$this->adapter->setNamespace('utopia_usage_schema');
31+
$this->adapter->setSharedTables(true);
32+
$this->adapter->setTenant('1');
33+
34+
if ($database = getenv('CLICKHOUSE_DATABASE')) {
35+
$this->adapter->setDatabase($database);
36+
}
37+
38+
$usage = new Usage($this->adapter);
39+
$usage->setup();
40+
}
41+
42+
public function testEventsTableCarriesCodecsAndLowCardinality(): void
43+
{
44+
$ddl = $this->showCreate($this->resolveTableName('getEventsTableName'));
45+
46+
$this->assertStringContainsString('`time` DateTime64(3) CODEC(Delta(4), LZ4)', $ddl);
47+
$this->assertStringContainsString('`id` String CODEC(ZSTD(3))', $ddl);
48+
$this->assertStringContainsString('`path` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl);
49+
$this->assertStringContainsString('`hostname` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl);
50+
$this->assertStringContainsString('`resourceId` Nullable(String) CODEC(ZSTD(3))', $ddl);
51+
$this->assertStringContainsString('`teamId` Nullable(String) CODEC(ZSTD(3))', $ddl);
52+
$this->assertStringContainsString('`osVersion` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl);
53+
$this->assertStringContainsString('`deviceModel` LowCardinality(Nullable(String)) CODEC(ZSTD(3))', $ddl);
54+
}
55+
56+
public function testEventsTableSwapsBloomForSetOnLowCardinality(): void
57+
{
58+
$ddl = $this->showCreate($this->resolveTableName('getEventsTableName'));
59+
60+
// status / method / country / service / clientType / osName must be set(0)
61+
$this->assertStringContainsString('`index-status` status TYPE set(0)', $ddl);
62+
$this->assertStringContainsString('`index-method` method TYPE set(0)', $ddl);
63+
$this->assertStringContainsString('`index-country` country TYPE set(0)', $ddl);
64+
$this->assertStringContainsString('`index-service` service TYPE set(0)', $ddl);
65+
$this->assertStringContainsString('`index-clientType` clientType TYPE set(0)', $ddl);
66+
$this->assertStringContainsString('`index-osName` osName TYPE set(0)', $ddl);
67+
68+
// path / hostname / resourceId / teamId stay bloom_filter
69+
$this->assertStringContainsString('`index-path` path TYPE bloom_filter', $ddl);
70+
$this->assertStringContainsString('`index-hostname` hostname TYPE bloom_filter', $ddl);
71+
$this->assertStringContainsString('`index-resourceId` resourceId TYPE bloom_filter', $ddl);
72+
$this->assertStringContainsString('`index-teamId` teamId TYPE bloom_filter', $ddl);
73+
}
74+
75+
public function testDailyTableCarriesCodecs(): void
76+
{
77+
$ddl = $this->showCreate($this->resolveTableName('getEventsDailyTableName'));
78+
79+
$this->assertStringContainsString('`time` DateTime64(3) CODEC(Delta(4), LZ4)', $ddl);
80+
$this->assertStringContainsString('`resourceId` Nullable(String) CODEC(ZSTD(3))', $ddl);
81+
$this->assertStringContainsString('`teamInternalId` Nullable(String) CODEC(ZSTD(3))', $ddl);
82+
}
83+
84+
private function resolveTableName(string $accessor): string
85+
{
86+
$reflection = new ReflectionClass($this->adapter);
87+
$method = $reflection->getMethod($accessor);
88+
$method->setAccessible(true);
89+
$raw = $method->invoke($this->adapter);
90+
return is_string($raw) ? $raw : '';
91+
}
92+
93+
private function showCreate(string $table): string
94+
{
95+
$reflection = new ReflectionClass($this->adapter);
96+
$dbProp = $reflection->getProperty('database');
97+
$dbProp->setAccessible(true);
98+
$dbValue = $dbProp->getValue($this->adapter);
99+
$database = is_string($dbValue) ? $dbValue : '';
100+
101+
$sql = "SHOW CREATE TABLE `{$database}`.`{$table}` FORMAT JSON";
102+
$query = $reflection->getMethod('query');
103+
$query->setAccessible(true);
104+
$raw = $query->invoke($this->adapter, $sql, []);
105+
$rawString = is_string($raw) ? $raw : '';
106+
$json = json_decode($rawString, true);
107+
if (is_array($json) && isset($json['data'][0]) && is_array($json['data'][0])) {
108+
$row = $json['data'][0];
109+
$statement = $row['statement'] ?? '';
110+
if (is_string($statement)) {
111+
return $statement;
112+
}
113+
}
114+
return '';
115+
}
116+
}

0 commit comments

Comments
 (0)