fix analytics queries (#1141)

<!--

Make sure you've read the CONTRIBUTING.md guidelines:
https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md

-->


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Improvements**
* Added automatic retry with stricter result validation and clearer
error handling for query profiling to improve reliability.

* **Chores**
* CI workflows updated to use larger runner instances (upgraded runner
size across relevant jobs).

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: BilalG1

.github/workflows/restart-dev-and-test-with-custom-base-port.yaml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   restart-dev-and-test-with-custom-base-port:
-    runs-on: ubicloud-standard-8
+    runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
 

.github/workflows/restart-dev-and-test.yaml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   restart-dev-and-test:
-    runs-on: ubicloud-standard-8
+    runs-on: ubicloud-standard-16
 
     steps:
       - uses: actions/checkout@v6

.github/workflows/setup-tests-with-custom-base-port.yaml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   setup-tests-with-custom-base-port:
-    runs-on: ubicloud-standard-8
+    runs-on: ubicloud-standard-16
     env:
       NEXT_PUBLIC_STACK_PORT_PREFIX: "69"
 

.github/workflows/setup-tests.yaml

@@ -16,7 +16,7 @@ env:
 
 jobs:
   setup-tests:
-    runs-on: ubicloud-standard-8
+    runs-on: ubicloud-standard-16
     steps:
       - uses: actions/checkout@v6
 

apps/backend/src/lib/clickhouse.tsx

@@ -48,30 +48,44 @@ export const getQueryTimingStats = async (client: ClickHouseClient, queryId: str
       password: clickhouseAdminPassword,
     },
   });
-  const profile = await client.query({
-    query: `
-    SELECT
-      ProfileEvents['CPUTimeMicroseconds'] / 1000 AS cpu_time_ms,
-      ProfileEvents['RealTimeMicroseconds'] / 1000 AS wall_clock_time_ms
-    FROM system.query_log
-    WHERE query_id = {query_id:String} AND type = 'QueryFinish'
-    ORDER BY event_time DESC
-    LIMIT 1
-  `,
-    query_params: { query_id: queryId },
-    auth: {
-      username: clickhouseAdminUser,
-      password: clickhouseAdminPassword,
-    },
-    format: "JSON",
-  });
+  const queryProfile = async () => {
+    const profile = await client.query({
+      query: `
+      SELECT
+        ProfileEvents['CPUTimeMicroseconds'] / 1000 AS cpu_time_ms,
+        ProfileEvents['RealTimeMicroseconds'] / 1000 AS wall_clock_time_ms
+      FROM system.query_log
+      WHERE query_id = {query_id:String} AND type = 'QueryFinish'
+      ORDER BY event_time DESC
+      LIMIT 1
+    `,
+      query_params: { query_id: queryId },
+      auth: {
+        username: clickhouseAdminUser,
+        password: clickhouseAdminPassword,
+      },
+      format: "JSON",
+    });
+
+    return await profile.json<{
+      cpu_time_ms: number,
+      wall_clock_time_ms: number,
+    }>();
+  };
 
-  const stats = await profile.json<{
-    cpu_time_ms: number,
-    wall_clock_time_ms: number,
-  }>();
-  if (stats.data.length !== 1) {
-    throw new StackAssertionError(`Unexpected number of query log results: ${stats.data.length}`, { data: stats.data });
+  const retryDelaysMs = [75, 150, 300, 600, 1200];
+  for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) {
+    const stats = await queryProfile();
+    if (stats.data.length === 1) {
+      return stats.data[0];
+    }
+    if (stats.data.length > 1) {
+      throw new StackAssertionError(`Unexpected number of query log results: ${stats.data.length}`, { data: stats.data });
+    }
+    if (attempt < retryDelaysMs.length) {
+      await new Promise((resolve) => setTimeout(resolve, retryDelaysMs[attempt]));
+    }
   }
-  return stats.data[0];
+
+  throw new StackAssertionError("Unexpected number of query log results: 0", { data: [] });
 };