Skip to content

Commit 8868360

Browse files
committed
fix(db): enforce ClickHouse schema parity
1 parent 6b9946c commit 8868360

15 files changed

Lines changed: 301 additions & 136 deletions

File tree

.github/workflows/ci.yml

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,6 @@ jobs:
4343
name: Check Types
4444
runs-on: blacksmith-4vcpu-ubuntu-2404
4545
timeout-minutes: 15
46-
env:
47-
CLICKHOUSE_READONLY_URL: ${{ secrets.CLICKHOUSE_READONLY_URL }}
4846
steps:
4947
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
5048
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -61,8 +59,12 @@ jobs:
6159
- name: ClickHouse types match DDL
6260
run: bun run --cwd packages/db ch:check
6361
- name: ClickHouse schema matches cluster
64-
if: env.CLICKHOUSE_READONLY_URL != ''
65-
run: bun run --cwd packages/db ch:verify
62+
env:
63+
CLICKHOUSE_READONLY_URL: ${{ secrets.CLICKHOUSE_READONLY_URL }}
64+
run: |
65+
if [[ -n "$CLICKHOUSE_READONLY_URL" ]]; then
66+
bun run --cwd packages/db ch:verify
67+
fi
6668
6769
test:
6870
name: Test
@@ -127,10 +129,12 @@ jobs:
127129
ports:
128130
- 8123:8123
129131
options: >-
132+
--ulimit nofile=262144:262144
130133
--health-cmd "clickhouse-client --query 'SELECT 1'"
131134
--health-interval 10s
132135
--health-timeout 5s
133-
--health-retries 5
136+
--health-start-period 30s
137+
--health-retries 12
134138
135139
steps:
136140
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -158,6 +162,8 @@ jobs:
158162
run: bun run --cwd packages/db clickhouse:init
159163
- name: Test
160164
env:
165+
CLICKHOUSE_INTEGRATION_TESTS: "true"
166+
CLICKHOUSE_URL: http://default:@localhost:8123
161167
NODE_ENV: test
162168
run: bun run test
163169
- name: Insights integration

lefthook.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ output:
77
- execution_out
88

99
pre-commit:
10-
parallel: true
10+
parallel: false
1111
commands:
1212
no-secrets:
1313
glob:
@@ -24,15 +24,15 @@ pre-commit:
2424
run: |
2525
blocked_files="$(echo "{staged_files}" | tr ' ' '\n' | grep -vE '(^|/)\.env\.example$' || true)"
2626
[ -z "$blocked_files" ] || (echo "Blocked: staged file(s) may contain secrets:" && echo "$blocked_files" | sed 's/^/ - /' && exit 1)
27-
check-types:
28-
glob: "**/*.{ts,tsx}"
29-
run: bun run check-types
3027
ch-types:
3128
glob: "packages/db/src/clickhouse/schema/**/*.sql"
3229
run: |
3330
set -e
3431
bun run generate-db
3532
git add packages/db/src/clickhouse/schema/tables.generated.ts
33+
check-types:
34+
glob: "**/*.{ts,tsx}"
35+
run: bun run check-types
3636
lockfile-check:
3737
glob:
3838
- "**/package.json"

packages/ai/src/types/tables.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,13 @@ import type {
66
OutgoingLinksRow,
77
RevenueRow,
88
UptimeMonitorRow,
9-
WebVitalsHourlyRow,
109
WebVitalsSpansRow,
1110
} from "@databuddy/db/clickhouse/tables";
1211

1312
export const Analytics = {
1413
events: "analytics.events",
1514
error_spans: "analytics.error_spans",
1615
web_vitals_spans: "analytics.web_vitals_spans",
17-
web_vitals_hourly: "analytics.web_vitals_hourly",
1816
custom_events: "analytics.custom_events",
1917
blocked_traffic: "analytics.blocked_traffic",
2018
outgoing_links: "analytics.outgoing_links",
@@ -32,7 +30,6 @@ export interface TableFieldsMap {
3230
"analytics.events": keyof EventsRow;
3331
"analytics.outgoing_links": keyof OutgoingLinksRow;
3432
"analytics.revenue": keyof RevenueRow;
35-
"analytics.web_vitals_hourly": keyof WebVitalsHourlyRow;
3633
"analytics.web_vitals_spans": keyof WebVitalsSpansRow;
3734
"uptime.uptime_monitor": keyof UptimeMonitorRow;
3835
}

packages/db/src/clickhouse/apply.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@ import { readSql, sqlFiles } from "./schema-parse";
66
const SCHEMA_DIR = join(dirname(fileURLToPath(import.meta.url)), "schema");
77

88
const SINGLE_NODE = process.env.CLICKHOUSE_CLUSTER == null;
9+
const CREATE_DATABASE_PATTERN =
10+
/CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\./i;
911

1012
function databaseOf(sql: string): string {
11-
const m = sql.match(
12-
/CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\./i
13-
);
13+
const m = sql.match(CREATE_DATABASE_PATTERN);
1414
if (!m) {
15-
throw new Error(`Could not determine database for statement: ${sql.slice(0, 80)}`);
15+
throw new Error(
16+
`Could not determine database for statement: ${sql.slice(0, 80)}`
17+
);
1618
}
1719
return m[1];
1820
}
@@ -33,7 +35,9 @@ export async function applyClickHouseSchema(): Promise<{
3335
const tables = files.filter((f) => !f.endsWith("_mv.sql"));
3436
const views = files.filter((f) => f.endsWith("_mv.sql"));
3537

36-
const databases = [...new Set(files.map((f) => databaseOf(readSql(f))))].sort();
38+
const databases = [
39+
...new Set(files.map((f) => databaseOf(readSql(f)))),
40+
].sort();
3741
for (const db of databases) {
3842
await clickHouse.command({ query: `CREATE DATABASE IF NOT EXISTS ${db}` });
3943
}

packages/db/src/clickhouse/codegen.ts

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,34 @@ import {
1313

1414
const SCHEMA_DIR = join(dirname(fileURLToPath(import.meta.url)), "schema");
1515
const OUT_FILE = join(SCHEMA_DIR, "tables.generated.ts");
16+
const ARRAY_PATTERN = /^Array\((.*)\)$/i;
17+
const BOOLEAN_PATTERN = /^Bool/i;
18+
const DATE_PATTERN = /^(?:Date|DateTime)/i;
19+
const FLOAT_PATTERN = /^Float/i;
20+
const INTEGER_PATTERN = /Int/i;
21+
const LOW_CARDINALITY_PATTERN = /^LowCardinality\((.*)\)$/i;
22+
const NULLABLE_PATTERN = /^Nullable\((.*)\)$/i;
1623

1724
function tsType(rawType: string): string {
1825
let t = rawType.trim();
1926
let nullable = false;
20-
const lc = t.match(/^LowCardinality\((.*)\)$/i);
27+
const lc = t.match(LOW_CARDINALITY_PATTERN);
2128
if (lc) {
2229
t = lc[1].trim();
2330
}
24-
const nul = t.match(/^Nullable\((.*)\)$/i);
31+
const nul = t.match(NULLABLE_PATTERN);
2532
if (nul) {
2633
nullable = true;
2734
t = nul[1].trim();
2835
}
29-
const arr = t.match(/^Array\((.*)\)$/i);
36+
const arr = t.match(ARRAY_PATTERN);
3037
if (arr) {
3138
return `${tsType(arr[1])}[]`;
3239
}
3340
let base: string;
34-
if (/Int/i.test(t) || /^Float/i.test(t)) {
41+
if (INTEGER_PATTERN.test(t) || FLOAT_PATTERN.test(t)) {
3542
base = "number";
36-
} else if (/^Bool/i.test(t)) {
43+
} else if (BOOLEAN_PATTERN.test(t)) {
3744
base = "boolean";
3845
} else {
3946
base = "string";
@@ -49,12 +56,12 @@ function pascal(name: string): string {
4956
}
5057

5158
function insertTsType(rawType: string): string {
52-
let t = rawType.replace(/^LowCardinality\((.*)\)$/i, "$1").trim();
53-
const nul = t.match(/^Nullable\((.*)\)$/i);
59+
let t = rawType.replace(LOW_CARDINALITY_PATTERN, "$1").trim();
60+
const nul = t.match(NULLABLE_PATTERN);
5461
if (nul) {
5562
t = nul[1].trim();
5663
}
57-
if (/^(?:Date|DateTime)/i.test(t)) {
64+
if (DATE_PATTERN.test(t)) {
5865
const base = "number | string";
5966
return isNullable(rawType) ? `${base} | null` : base;
6067
}
@@ -113,5 +120,8 @@ const columnsConst = `export const TABLE_COLUMNS = {\n${tables
113120
)
114121
.join("\n")}\n} as const satisfies Record<string, readonly string[]>;`;
115122

116-
writeFileSync(OUT_FILE, `${header}\n${interfaces}\n\n${registry}\n\n${columnsConst}\n`);
123+
writeFileSync(
124+
OUT_FILE,
125+
`${header}\n${interfaces}\n\n${registry}\n\n${columnsConst}\n`
126+
);
117127
console.info(`Generated ${OUT_FILE} (${tables.length} tables)`);
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import { describe, expect, it } from "bun:test";
2+
import { parseTable } from "./schema-parse";
3+
4+
describe("parseTable indexes", () => {
5+
it("captures normalized secondary index definitions", () => {
6+
const table = parseTable(`
7+
CREATE TABLE IF NOT EXISTS analytics.example
8+
(
9+
\`client_id\` String,
10+
\`timestamp\` DateTime64(3, 'UTC'),
11+
INDEX \`idx_client_id\`
12+
client_id TYPE bloom_filter(0.01) GRANULARITY 1,
13+
INDEX idx_timestamp timestamp TYPE minmax GRANULARITY 2
14+
)
15+
ENGINE = MergeTree
16+
ORDER BY (client_id, timestamp)
17+
`);
18+
19+
expect(table.indexes).toEqual([
20+
{
21+
name: "idx_client_id",
22+
definition: "client_id TYPE bloom_filter(0.01) GRANULARITY 1",
23+
},
24+
{
25+
name: "idx_timestamp",
26+
definition: "timestamp TYPE minmax GRANULARITY 2",
27+
},
28+
]);
29+
});
30+
31+
it("does not treat indexes as columns", () => {
32+
const table = parseTable(`
33+
CREATE TABLE analytics.example
34+
(
35+
client_id String,
36+
INDEX idx_client_id client_id TYPE set(100) GRANULARITY 1
37+
)
38+
ENGINE = MergeTree
39+
ORDER BY client_id
40+
`);
41+
42+
expect(table.columns.map((column) => column.name)).toEqual(["client_id"]);
43+
expect(table.indexes).toHaveLength(1);
44+
});
45+
});

packages/db/src/clickhouse/schema-parse.ts

Lines changed: 58 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,44 @@
11
import { readdirSync, readFileSync, statSync } from "node:fs";
22
import { join } from "node:path";
33

4+
const COLUMN_DEFAULT_PATTERN = /\b(?:DEFAULT|EPHEMERAL)\b/i;
5+
const COLUMN_NAME_PATTERN = /^`?(\w+)`?\s+/;
6+
const COMPUTED_COLUMN_PATTERN = /\b(?:MATERIALIZED|ALIAS)\b/i;
7+
const INDEX_NAME_PATTERN = /^INDEX\s+`?(\w+)`?\s+/i;
8+
const LOW_CARDINALITY_PATTERN = /^LowCardinality\((.*)\)$/i;
9+
const MATERIALIZED_VIEW_PATTERN = /MATERIALIZED\s+VIEW/i;
10+
const NULLABLE_PATTERN = /^Nullable\(/i;
11+
const QUALIFIED_NAME_PATTERN =
12+
/CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:`([^`]+)`|(\w+))\.(?:`([^`]+)`|(\w+))/i;
13+
const TABLE_NAME_PATTERN =
14+
/CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?[\w.]*\.(\w+)/i;
15+
416
export interface ParsedColumn {
17+
computed: boolean;
18+
hasDefault: boolean;
519
name: string;
6-
type: string;
720
nullable: boolean;
8-
hasDefault: boolean;
9-
computed: boolean;
21+
type: string;
22+
}
23+
24+
export interface ParsedIndex {
25+
definition: string;
26+
name: string;
1027
}
1128

1229
export function isNullable(type: string): boolean {
13-
const inner = type.replace(/^LowCardinality\((.*)\)$/i, "$1").trim();
14-
return /^Nullable\(/i.test(inner);
30+
const inner = type.replace(LOW_CARDINALITY_PATTERN, "$1").trim();
31+
return NULLABLE_PATTERN.test(inner);
1532
}
1633

1734
export interface ParsedTable {
18-
name: string;
19-
isView: boolean;
2035
columns: ParsedColumn[];
2136
engine: string;
22-
partitionBy: string;
37+
indexes: ParsedIndex[];
38+
isView: boolean;
39+
name: string;
2340
orderBy: string;
41+
partitionBy: string;
2442
settings: string;
2543
}
2644

@@ -85,19 +103,15 @@ const COLUMN_MODIFIERS =
85103
const NON_COLUMN = /^(?:INDEX|CONSTRAINT|PROJECTION|PRIMARY\s+KEY)\b/i;
86104

87105
export function tableNameOf(sql: string): string {
88-
const m = sql.match(
89-
/CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?[\w.]*\.(\w+)/i
90-
);
106+
const m = sql.match(TABLE_NAME_PATTERN);
91107
if (!m) {
92108
throw new Error("Could not parse table name");
93109
}
94110
return m[1];
95111
}
96112

97113
export function qualifiedNameOf(sql: string): string {
98-
const m = sql.match(
99-
/CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:`([^`]+)`|(\w+))\.(?:`([^`]+)`|(\w+))/i
100-
);
114+
const m = sql.match(QUALIFIED_NAME_PATTERN);
101115
if (!m) {
102116
throw new Error("Could not parse qualified table name");
103117
}
@@ -112,7 +126,7 @@ export function parseColumns(sql: string): ParsedColumn[] {
112126
if (NON_COLUMN.test(item)) {
113127
continue;
114128
}
115-
const nameMatch = item.match(/^`?(\w+)`?\s+/);
129+
const nameMatch = item.match(COLUMN_NAME_PATTERN);
116130
if (!nameMatch) {
117131
continue;
118132
}
@@ -123,31 +137,53 @@ export function parseColumns(sql: string): ParsedColumn[] {
123137
name: nameMatch[1],
124138
type,
125139
nullable: isNullable(type),
126-
hasDefault: /\b(?:DEFAULT|EPHEMERAL)\b/i.test(afterName),
127-
computed: /\b(?:MATERIALIZED|ALIAS)\b/i.test(afterName),
140+
hasDefault: COLUMN_DEFAULT_PATTERN.test(afterName),
141+
computed: COMPUTED_COLUMN_PATTERN.test(afterName),
128142
});
129143
}
130144
return cols;
131145
}
132146

147+
function normalizeDefinition(definition: string): string {
148+
return definition.replaceAll("`", "").replace(/\s+/g, " ").trim();
149+
}
150+
151+
export function parseIndexes(sql: string): ParsedIndex[] {
152+
const indexes: ParsedIndex[] = [];
153+
for (const item of splitTopLevel(firstParenGroup(sql).body)) {
154+
const match = item.match(INDEX_NAME_PATTERN);
155+
if (!match) {
156+
continue;
157+
}
158+
indexes.push({
159+
name: match[1],
160+
definition: normalizeDefinition(item.slice(match[0].length)),
161+
});
162+
}
163+
return indexes;
164+
}
165+
133166
function clause(tail: string, keyword: string, stops: string[]): string {
134-
const lookahead = stops.length
135-
? `(?=(?:${stops.join("|")})\\b|$)`
136-
: "(?=$)";
167+
const lookahead = stops.length ? `(?=(?:${stops.join("|")})\\b|$)` : "(?=$)";
137168
const re = new RegExp(`${keyword}\\s+([\\s\\S]*?)\\s*${lookahead}`, "i");
138169
const m = tail.match(re);
139170
return m ? m[1].trim() : "";
140171
}
141172

142173
export function parseTable(sql: string): ParsedTable {
143174
const name = tableNameOf(sql);
144-
const isView = /MATERIALIZED\s+VIEW/i.test(sql);
175+
const isView = MATERIALIZED_VIEW_PATTERN.test(sql);
145176
const columns = parseColumns(sql);
146-
const tail = sql.slice(firstParenGroup(sql).end + 1).replace(/\s+/g, " ").trim();
177+
const indexes = parseIndexes(sql);
178+
const tail = sql
179+
.slice(firstParenGroup(sql).end + 1)
180+
.replace(/\s+/g, " ")
181+
.trim();
147182
return {
148183
name,
149184
isView,
150185
columns,
186+
indexes,
151187
engine: clause(tail, "ENGINE\\s*=", [
152188
"PARTITION BY",
153189
"PRIMARY KEY",

0 commit comments

Comments
 (0)