clickhouse setup

.github/workflows/docker-server-test.yaml

@@ -23,6 +23,12 @@ jobs:
           sleep 5
           docker logs db
 
+      - name: Setup clickhouse
+        run: |
+          docker run -d --name clickhouse -e CLICKHOUSE_DB=analytics -e CLICKHOUSE_USER=stackframe -e CLICKHOUSE_PASSWORD=password -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -p 8133:8123 clickhouse/clickhouse-server:25.10
+          sleep 5
+          docker logs clickhouse
+
       - name: Build Docker image
         run: docker build -f docker/server/Dockerfile -t server .
 

apps/backend/.env.development

@@ -70,3 +70,11 @@ STACK_QSTASH_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}25
 STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc3N3b3JkIn0=
 STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r
 STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs
+
+# Clickhouse
+STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}33
+STACK_CLICKHOUSE_ADMIN_USER=stackframe
+STACK_CLICKHOUSE_ADMIN_PASSWORD=PASSWORD-PLACEHOLDER--9gKyMxJeMx
+STACK_CLICKHOUSE_EXTERNAL_USER=limited_user
+STACK_CLICKHOUSE_EXTERNAL_PASSWORD=PASSWORD-PLACEHOLDER--EZeHscBMzE
+STACK_CLICKHOUSE_DATABASE=analytics

apps/backend/package.json

@@ -43,6 +43,7 @@
   "dependencies": {
     "@ai-sdk/openai": "^1.3.23",
     "@aws-sdk/client-s3": "^3.855.0",
+    "@clickhouse/client": "^1.14.0",
     "@next/bundle-analyzer": "15.2.3",
     "@node-oauth/oauth2-server": "^5.1.0",
     "@opentelemetry/api": "^1.9.0",

apps/backend/scripts/clickhouse-migrations.ts

@@ -0,0 +1,39 @@
+import { createClickhouseClient } from "@/lib/clickhouse";
+import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
+
+export async function runClickhouseMigrations() {
+  console.log("Running Clickhouse migrations...");
+  const client = createClickhouseClient("admin");
+  const clickhouseExternalPassword = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_PASSWORD");
+  // todo: create migration files
+  await client.exec({
+    query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH plaintext_password BY {clickhouseExternalPassword:String}",
+    query_params: { clickhouseExternalPassword },
+  });
+  const queries = [
+    "GRANT SELECT ON analytics.allowed_table1 TO limited_user;",
+    "REVOKE ALL ON system.* FROM limited_user;",
+    "REVOKE CREATE, ALTER, DROP, INSERT ON *.* FROM limited_user;"
+  ];
+  for (const query of queries) {
+    console.log(query);
+    await client.exec({ query });
+  }
+  console.log("Clickhouse migrations complete");
+  await client.close();
+}
+
+
+
+const EVENTS_TABLE_BASE_SQL = `
+CREATE TABLE IF NOT EXISTS events (
+    event_id    UUID DEFAULT generateUUIDv4(),
+    event_type  LowCardinality(String),
+    event_at    DateTime64(3, 'UTC'),
+    data        JSON,
+    created_at DateTime64(3, 'UTC') DEFAULT now64(3)
+)
+ENGINE MergeTree
+PARTITION BY toYYYYMM(event_at)
+ORDER BY (event_at, event_type);
+`;

apps/backend/scripts/db-migrations.ts

@@ -8,6 +8,7 @@ import path from "path";
 import * as readline from "readline";
 import { seed } from "../prisma/seed";
 import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
+import { runClickhouseMigrations } from "./clickhouse-migrations";
 
 const dropSchema = async () => {
   await globalPrismaClient.$executeRaw(Prisma.sql`DROP SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} CASCADE`);
@@ -151,6 +152,8 @@ const migrate = async (selectedMigrationFiles?: { migrationName: string, sql: st
 
   console.log('='.repeat(60) + '\n');
 
+  await runClickhouseMigrations();
+
   return result;
 };
 

apps/backend/src/app/api/latest/analytics/query/route.tsx

@@ -0,0 +1,74 @@
+import { createClickhouseClient, getQueryTimingStats } from "@/lib/clickhouse";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { KnownErrors } from "@stackframe/stack-shared";
+import { adaptSchema, jsonSchema, serverOrHigherAuthTypeSchema, yupBoolean, yupMixed, yupNumber, yupObject, yupRecord, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { Result } from "@stackframe/stack-shared/dist/utils/results";
+import { randomUUID } from "crypto";
+
+export const POST = createSmartRouteHandler({
+  metadata: {
+    summary: "Execute analytics query",
+    description: "Execute a ClickHouse query against the analytics database",
+    tags: ["Analytics"],
+  },
+  request: yupObject({
+    auth: yupObject({
+      type: serverOrHigherAuthTypeSchema,
+      tenancy: adaptSchema,
+    }).defined(),
+    body: yupObject({
+      include_all_branches: yupBoolean().default(false),
+      query: yupString().defined().nonEmpty(),
+      params: yupRecord(yupString().defined(), yupMixed().defined()).default({}),
+      timeout_ms: yupNumber().integer().min(1).max(60000).default(1000),
+    }).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: yupObject({
+      result: jsonSchema.defined(),
+      stats: yupObject({
+        cpu_time: yupNumber().defined(),
+        wall_clock_time: yupNumber().defined(),
+      }).defined(),
+    }).defined(),
+  }),
+  async handler({ body, auth }) {
+    const client = createClickhouseClient("external", body.timeout_ms);
+    const queryId = randomUUID();
+    const resultSet = await Result.fromPromise(client.query({
+      query: body.query,
+      query_id: queryId,
+      query_params: body.params,
+      clickhouse_settings: {
+        SQL_tenancy_id: auth.tenancy.id,
+      },
+      format: "JSONEachRow",
+    }));
+
+    if (resultSet.status === "error") {
+      const message = resultSet.error instanceof Error ? resultSet.error.message : null;
+      if (message === "Timeout error.") {
+        throw new KnownErrors.AnalyticsQueryTimeout(body.timeout_ms);
+      }
+      throw new KnownErrors.AnalyticsQueryError(message ?? "Unknown error");
+    }
+
+    const rows = await resultSet.data.json<Record<string, unknown>[]>();
+    const stats = await getQueryTimingStats(client, queryId);
+
+    return {
+      statusCode: 200,
+      bodyType: "json",
+      body: {
+        result: rows,
+        stats: {
+          cpu_time: stats.cpu_time_ms,
+          wall_clock_time: stats.wall_clock_time_ms,
+        },
+      },
+    };
+  },
+});
+

apps/backend/src/lib/clickhouse.tsx

@@ -0,0 +1,56 @@
+import { createClient, type ClickHouseClient } from "@clickhouse/client";
+import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
+
+const clickhouseUrl = getEnvVariable("STACK_CLICKHOUSE_URL");
+const clickhouseAdminUser = getEnvVariable("STACK_CLICKHOUSE_ADMIN_USER", "stackframe");
+const clickhouseExternalUser = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_USER", "limited_user");
+const clickhouseAdminPassword = getEnvVariable("STACK_CLICKHOUSE_ADMIN_PASSWORD");
+const clickhouseExternalPassword = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_PASSWORD");
+const clickhouseDatabase = getEnvVariable("STACK_CLICKHOUSE_DATABASE", "analytics");
+
+export function createClickhouseClient(authType: "admin" | "external", timeoutMs?: number) {
+  return createClient({
+    url: clickhouseUrl,
+    username: authType === "admin" ? clickhouseAdminUser : clickhouseExternalUser,
+    password: authType === "admin" ? clickhouseAdminPassword : clickhouseExternalPassword,
+    database: clickhouseDatabase,
+    request_timeout: timeoutMs,
+  });
+}
+
+
+export const getQueryTimingStats = async (client: ClickHouseClient, queryId: string) => {
+  // Flush logs to ensure system.query_log has latest query result.
+  // Todo: for performance we should instead poll for this row to become available asynchronously after returning result. Flushed every 7.5 seconds by default
+  await client.exec({
+    query: "SYSTEM FLUSH LOGS",
+    auth: {
+      username: clickhouseAdminUser,
+      password: clickhouseAdminPassword,
+    },
+  });
+  const profile = await client.query({
+    query: `
+    SELECT
+      ProfileEvents['CPUTimeMicroseconds'] / 1000 AS cpu_time_ms,
+      ProfileEvents['RealTimeMicroseconds'] / 1000 AS wall_clock_time_ms
+    FROM system.query_log
+    WHERE query_id = {query_id:String} AND type = 'QueryFinish'
+    ORDER BY event_time DESC
+    LIMIT 1
+  `,
+    query_params: { query_id: queryId },
+    auth: {
+      username: clickhouseAdminUser,
+      password: clickhouseAdminPassword,
+    },
+    format: "JSON",
+  });
+
+  const stats = await profile.json<{
+    cpu_time_ms: number,
+    wall_clock_time_ms: number,
+  }>();
+  return stats.data[0];
+};
+

apps/dev-launchpad/public/index.html

@@ -129,6 +129,7 @@ <h2 style="margin-top: 64px;">Background services</h2>
         { suffix: "22", label: "Freestyle mock" },
         { suffix: "24", label: "LocalStack Gateway (AWS mock)" },
         { suffix: "25", label: "QStash mock" },
+        { suffix: "33", label: "ClickHouse" },
         { range: ["50", "99"], label: "Reserved for LocalStack (external services)" },
       ];
 
@@ -300,6 +301,23 @@ <h2 style="margin-top: 64px;">Background services</h2>
             "React example",
           ],
         },
+        {
+          name: "ClickHouse",
+          portSuffix: "33",
+          description: [
+            "ClickHouse",
+          ],
+          importance: 1,
+          img: "https://thumbs.bfldr.com/at/qkjfv3nvsv4rbwn94zmtb4t/v/1197417003?expiry=1764357242&fit=bounds&height=800&sig=NjEwNzA0OThjZmJiZDQzZmUwNjIyY2UxYzZiNGYxNmQ3NjJiYjc0OA%3D%3D&width=1100",
+        },
+        {
+          name: "MCPJam Inspector",
+          portSuffix: "26",
+          importance: 1,
+          description: [
+            "MCP tool inspector",
+          ],
+        },
         {
           name: "Convex example",
           portSuffix: "27",