Skip to content

Commit 68a4b6f

Browse files
lohanidamodarlohanidamodar
committed
fix: re-key event/gauge tables on (tenant, metric, time, id) for time-range scan efficiency
Previous ORDER BY of (tenant, id) had id (random UUID) as the primary sort key, so ClickHouse stored rows in essentially random physical order. Time-range predicates like WHERE time > X had to scan every granule because the primary index had no time information to skip on. Re-key to (tenant, metric, time, id) so the primary index matches how the data is actually queried: - tenant: multi-tenant isolation (cheap first-level filter) - metric: per-metric series (most queries are scoped to one) - time: range scans now hit a small contiguous span instead of the whole table - id: tiebreaker for stable physical ordering Gauges get the same shape. Daily MV already had the right key. Drop the now-redundant bloom_filter indexes on metric and time (primary key already covers them). Pre-prod schema change — no migration path needed, just DROP+CREATE on next deploy. Updates MetricTest counts to match the trimmed index lists.
1 parent b2cbdbc commit 68a4b6f

4 files changed

Lines changed: 32 additions & 55 deletions

File tree

‎.phpunit.result.cache‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":1,"defects":[],"times":{"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalCreation":0.001,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalAllValidIntervals":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidIntervalEmpty":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testIsGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByIntervalReturnsNullWhenMissing":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testRemoveGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testValidIntervalsConstant":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testUsageQueryExtendsQuery":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventSchemaReturnsAttributeDefinitions":0.001,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeSchemaReturnsAttributeDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetSchemaReturnsEventSchema":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIndexesReturnsEventIndexes":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidEventData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidGaugeData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsMinimalData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonStringMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsOversizedMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonIntegerValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDateTimeForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDatetimeStringForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsInvalidDatetimeString":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsEmptyTags":0,"Utopia\\Tests\\Usage\\MetricTest::testConstructorInitializesWithData":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsMetricId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsEmptyStringWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetMetricReturnsMetricName":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsType":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsDefaultType":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnCorrectValues":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsTimestamp":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsTags":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsEmptyArrayWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsTenantId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantConvertsNumericToString":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributesReturnsAllAttributes":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSetsAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSupportsChaining":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsTrueWhenExists":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsFalseWhenNotExists":0,"Utopia\\Tests\\Usage\\MetricTest::testRemoveAttributeRemovesAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsFalseWhenIdSet":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsTrueWhenNoId":0,"Utopia\\Tests\\Usage\\MetricTest::testToArrayReturnsArray":0,"Utopia\\Tests\\Usage\\MetricTest::testEventColumnsConstant":0}}
1+
{"version":1,"defects":{"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":3,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":3},"times":{"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalCreation":0.001,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalAllValidIntervals":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testGroupByIntervalInvalidIntervalEmpty":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testIsGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testExtractGroupByIntervalReturnsNullWhenMissing":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testRemoveGroupByInterval":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testValidIntervalsConstant":0,"Utopia\\Tests\\Usage\\UsageQueryTest::testUsageQueryExtendsQuery":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventSchemaReturnsAttributeDefinitions":0.001,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeSchemaReturnsAttributeDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetSchemaReturnsEventSchema":0,"Utopia\\Tests\\Usage\\MetricTest::testGetEventIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetGaugeIndexesReturnsIndexDefinitions":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIndexesReturnsEventIndexes":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidEventData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsValidGaugeData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsMinimalData":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsMissingValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonStringMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsOversizedMetric":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonIntegerValue":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDateTimeForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsDatetimeStringForTime":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsInvalidDatetimeString":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateRejectsNonArrayTags":0,"Utopia\\Tests\\Usage\\MetricTest::testValidateAcceptsEmptyTags":0,"Utopia\\Tests\\Usage\\MetricTest::testConstructorInitializesWithData":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsMetricId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetIdReturnsEmptyStringWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetMetricReturnsMetricName":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetValueReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsType":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTypeReturnsDefaultType":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testEventGettersReturnCorrectValues":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsTimestamp":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTimeReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsTags":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTagsReturnsEmptyArrayWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsTenantId":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantReturnsNullWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testGetTenantConvertsNumericToString":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributesReturnsAllAttributes":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsValue":0,"Utopia\\Tests\\Usage\\MetricTest::testGetAttributeReturnsDefaultWhenNotSet":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSetsAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testSetAttributeSupportsChaining":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsTrueWhenExists":0,"Utopia\\Tests\\Usage\\MetricTest::testHasAttributeReturnsFalseWhenNotExists":0,"Utopia\\Tests\\Usage\\MetricTest::testRemoveAttributeRemovesAndReturnsSelf":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsFalseWhenIdSet":0,"Utopia\\Tests\\Usage\\MetricTest::testIsEmptyReturnsTrueWhenNoId":0,"Utopia\\Tests\\Usage\\MetricTest::testToArrayReturnsArray":0,"Utopia\\Tests\\Usage\\MetricTest::testEventColumnsConstant":0}}

‎src/Usage/Adapter/ClickHouse.php‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,7 +1035,12 @@ private function createTable(string $tableName, string $type, array $indexes): v
10351035
$columnDefs = implode(",\n ", $columns);
10361036
$indexDefsStr = !empty($indexDefs) ? ",\n " . implode(",\n ", $indexDefs) : '';
10371037

1038-
$orderByExpr = $this->sharedTables ? '(tenant, id)' : '(id)';
1038+
// Primary key matches the most common filter pattern:
1039+
// tenant (multi-tenant isolation) → metric (per-metric series) →
1040+
// time (range scans). id is the tiebreaker for stable physical
1041+
// ordering. This shape lets ClickHouse skip whole granules on
1042+
// metric+time predicates instead of doing a full-table scan.
1043+
$orderByExpr = $this->sharedTables ? '(tenant, metric, time, id)' : '(metric, time, id)';
10391044

10401045
$createTableSql = "
10411046
CREATE TABLE IF NOT EXISTS {$escapedDatabaseAndTable} (
@@ -1073,19 +1078,15 @@ private function createDailyTable(): void
10731078
$columns[] = 'tenant Nullable(String)';
10741079
}
10751080

1076-
$indexes = [
1077-
'INDEX index_metric (metric) TYPE bloom_filter GRANULARITY 1',
1078-
'INDEX index_time (time) TYPE bloom_filter GRANULARITY 1',
1079-
];
1080-
10811081
$columnDefs = implode(",\n ", $columns);
1082-
$indexDefsStr = ",\n " . implode(",\n ", $indexes);
10831082

1083+
// metric and time are part of the ORDER BY (primary key) — no
1084+
// secondary bloom_filter indexes needed.
10841085
$dailyOrderBy = $this->sharedTables ? '(tenant, metric, time)' : '(metric, time)';
10851086

10861087
$createDailyTableSql = "
10871088
CREATE TABLE IF NOT EXISTS {$escapedDailyTable} (
1088-
{$columnDefs}{$indexDefsStr}
1089+
{$columnDefs}
10891090
)
10901091
ENGINE = SummingMergeTree()
10911092
ORDER BY {$dailyOrderBy}

‎src/Usage/Metric.php‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -583,17 +583,10 @@ public static function getSchema(): array
583583
*/
584584
public static function getEventIndexes(): array
585585
{
586+
// Note: `metric` and `time` are part of the ClickHouse primary key
587+
// (ORDER BY (tenant, metric, time, id)), so a separate bloom_filter
588+
// index on either column would be redundant. They're omitted here.
586589
return [
587-
[
588-
'$id' => 'index-metric',
589-
'type' => 'key',
590-
'attributes' => ['metric'],
591-
],
592-
[
593-
'$id' => 'index-time',
594-
'type' => 'key',
595-
'attributes' => ['time'],
596-
],
597590
[
598591
'$id' => 'index-path',
599592
'type' => 'key',
@@ -639,18 +632,10 @@ public static function getEventIndexes(): array
639632
*/
640633
public static function getGaugeIndexes(): array
641634
{
642-
return [
643-
[
644-
'$id' => 'index-metric',
645-
'type' => 'key',
646-
'attributes' => ['metric'],
647-
],
648-
[
649-
'$id' => 'index-time',
650-
'type' => 'key',
651-
'attributes' => ['time'],
652-
],
653-
];
635+
// `metric` and `time` are part of the primary key; no secondary
636+
// indexes needed. Returning an empty array keeps the table DDL
637+
// free of redundant bloom_filter clauses.
638+
return [];
654639
}
655640

656641
/**

‎tests/Usage/MetricTest.php‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -108,27 +108,19 @@ public function testGetEventIndexesReturnsIndexDefinitions(): void
108108
$indexes = Metric::getEventIndexes();
109109

110110
$this->assertIsArray($indexes);
111-
$this->assertCount(9, $indexes);
111+
// metric/time are now covered by the primary key (ORDER BY (tenant,
112+
// metric, time, id)), so only event-specific secondary indexes
113+
// remain: path, method, status, resource, resourceId, country,
114+
// userAgent.
115+
$this->assertCount(7, $indexes);
112116

113-
// Test metric index
114-
$metricIndex = $indexes[0];
115-
$this->assertEquals('index-metric', $metricIndex['$id']);
116-
$this->assertEquals('key', $metricIndex['type']);
117-
$this->assertEquals(['metric'], $metricIndex['attributes']);
118-
119-
// Test time index
120-
$timeIndex = $indexes[1];
121-
$this->assertEquals('index-time', $timeIndex['$id']);
122-
$this->assertEquals(['time'], $timeIndex['attributes']);
123-
124-
// Test event-specific indexes
125-
$this->assertEquals('index-path', $indexes[2]['$id']);
126-
$this->assertEquals('index-method', $indexes[3]['$id']);
127-
$this->assertEquals('index-status', $indexes[4]['$id']);
128-
$this->assertEquals('index-resource', $indexes[5]['$id']);
129-
$this->assertEquals('index-resourceId', $indexes[6]['$id']);
130-
$this->assertEquals('index-country', $indexes[7]['$id']);
131-
$this->assertEquals('index-userAgent', $indexes[8]['$id']);
117+
$this->assertEquals('index-path', $indexes[0]['$id']);
118+
$this->assertEquals('index-method', $indexes[1]['$id']);
119+
$this->assertEquals('index-status', $indexes[2]['$id']);
120+
$this->assertEquals('index-resource', $indexes[3]['$id']);
121+
$this->assertEquals('index-resourceId', $indexes[4]['$id']);
122+
$this->assertEquals('index-country', $indexes[5]['$id']);
123+
$this->assertEquals('index-userAgent', $indexes[6]['$id']);
132124
}
133125

134126
/**
@@ -138,11 +130,10 @@ public function testGetGaugeIndexesReturnsIndexDefinitions(): void
138130
{
139131
$indexes = Metric::getGaugeIndexes();
140132

133+
// Gauges only filter by metric and time, both in the primary key,
134+
// so no secondary indexes are needed.
141135
$this->assertIsArray($indexes);
142-
$this->assertCount(2, $indexes);
143-
144-
$this->assertEquals('index-metric', $indexes[0]['$id']);
145-
$this->assertEquals('index-time', $indexes[1]['$id']);
136+
$this->assertCount(0, $indexes);
146137
}
147138

148139
/**

0 commit comments

Comments
 (0)