fix(webapp): fail open on queue metrics and honor sparkline total override

The queues list tolerates a metrics query failure by rendering without metrics and logging a warning. UsageSparkline renders its total override even when every bucket is zero. The queue detail page returns 404 and its loader skips the metrics query when the feature flag is off. The seed script validates bucket size and only writes ClickHouse against a local host.

Author: ericallam

apps/webapp/app/components/primitives/UsageSparkline.tsx

@@ -50,7 +50,8 @@ export function UsageSparkline({
   formatTotal,
   totalClassName = "text-blue-400",
 }: UsageSparklineProps) {
-  if (!data || data.every((v) => v === 0)) {
+  const hasTotalOverride = totalOverride !== undefined;
+  if (!data || data.length === 0 || (data.every((v) => v === 0) && !hasTotalOverride)) {
     return <span className="text-text-dimmed">–</span>;
   }
 

apps/webapp/app/env.server.ts

@@ -899,15 +899,14 @@ const EnvironmentSchema = z
     QUEUE_METRICS_EMIT_ENABLED: z.string().default("0"),
     QUEUE_METRICS_CONSUMER_ENABLED: z.string().default("0"),
     QUEUE_METRICS_STREAM_SHARD_COUNT: z.coerce.number().int().default(4),
-    QUEUE_METRICS_STREAM_MAXLEN: z.coerce.number().int().default(2_000_000),
     QUEUE_METRICS_CONSUMER_BATCH_SIZE: z.coerce.number().int().default(1000),
     // Counter stream (exact counts, loss-intolerant). Unset host => the run-queue Redis;
     // set it to a dedicated instance so counter backlog never competes with the run queue.
     QUEUE_METRICS_REDIS_HOST: z.string().optional(),
     QUEUE_METRICS_REDIS_PORT: z.coerce.number().optional(),
     QUEUE_METRICS_REDIS_USERNAME: z.string().optional(),
     QUEUE_METRICS_REDIS_PASSWORD: z.string().optional(),
-    QUEUE_METRICS_REDIS_TLS_DISABLED: z.string().default("false"),
+    QUEUE_METRICS_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"),
     QUEUE_METRICS_COUNTER_STREAM_MAXLEN: z.coerce.number().int().default(8_000_000),
     // TTL (seconds) on the per-(queue,op) cumulative odometer key, refreshed on every write.
     // Idle-past-TTL queues purge and self-heal (restart from 1) on return; default 7 days.

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx

@@ -73,6 +73,7 @@ import {
 } from "~/presenters/v3/QueueMetricsPresenter.server";
 import { UsageSparkline } from "~/components/primitives/UsageSparkline";
 import { Area, AreaChart, ResponsiveContainer } from "recharts";
+import { logger } from "~/services/logger.server";
 import { requireUserId } from "~/services/session.server";
 import { cn } from "~/utils/cn";
 import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource";
@@ -160,19 +161,27 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     } | null = null;
 
     if (queueMetricsUiEnabled && queues.success) {
-      const presenter = new QueueMetricsPresenter();
-      const queueNames = queues.queues.map((q) => (q.type === "task" ? `task/${q.name}` : q.name));
-      const queueMetrics =
-        queueNames.length > 0
-          ? await presenter.getQueueListMetrics({ environment, queueNames, window: period })
-          : null;
-      if (queueMetrics) {
-        metrics = {
-          window: queueMetrics.window,
-          bucketStartMs: queueMetrics.bucketStartMs,
-          bucketIntervalMs: queueMetrics.bucketIntervalMs,
-          byQueue: Object.fromEntries(queueMetrics.byQueue),
-        };
+      // Metrics are additive observability; a ClickHouse hiccup must not take down queue
+      // management. Fail open to metrics: null instead of bubbling to the page-level 400.
+      try {
+        const presenter = new QueueMetricsPresenter();
+        const queueNames = queues.queues.map((q) =>
+          q.type === "task" ? `task/${q.name}` : q.name
+        );
+        const queueMetrics =
+          queueNames.length > 0
+            ? await presenter.getQueueListMetrics({ environment, queueNames, window: period })
+            : null;
+        if (queueMetrics) {
+          metrics = {
+            window: queueMetrics.window,
+            bucketStartMs: queueMetrics.bucketStartMs,
+            bucketIntervalMs: queueMetrics.bucketIntervalMs,
+            byQueue: Object.fromEntries(queueMetrics.byQueue),
+          };
+        }
+      } catch (error) {
+        logger.warn("Queue list metrics unavailable, rendering without them", { error });
       }
     }
 

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx

@@ -443,7 +443,7 @@ function QueueStats({
         loading={showLoading}
       />
       <Stat
-        label="Worst delay p95"
+        label="Delay p95"
         value={worstP95 > 0 ? formatWaitMs(worstP95) : "–"}
         loading={showLoading}
         className={worstP95 >= 60_000 ? "text-warning" : undefined}

apps/webapp/seed-queue-metrics.mts

@@ -282,7 +282,7 @@ function simulateBucket(
       env_running: envRunning,
       env_queued: envQueued,
       env_limit: envLimit,
-      throttled: running[q] >= limit[q] && queued[q] > 0 ? 1 : 0,
+      throttled: queued[q] > 0 && (running[q] >= limit[q] || scale < 1) ? 1 : 0,
     };
     rows.push(gauge);
 
@@ -317,11 +317,13 @@ function clickhouse(): ClickHouse {
     console.error("CLICKHOUSE_URL not set");
     process.exit(1);
   }
-  if (/\.clickhouse\.cloud|prod/i.test(clickhouseUrl)) {
-    console.error(`Refusing to run against a non-local ClickHouse: ${clickhouseUrl}`);
+  const url = new URL(clickhouseUrl);
+  // Allowlist local hosts only (this script TRUNCATEs), and never echo the URL (it carries creds).
+  const localHosts = new Set(["localhost", "127.0.0.1", "::1", "0.0.0.0"]);
+  if (!localHosts.has(url.hostname)) {
+    console.error(`Refusing to run against a non-local ClickHouse host: ${url.hostname}`);
     process.exit(1);
   }
-  const url = new URL(clickhouseUrl);
   url.searchParams.delete("secure");
   return new ClickHouse({ url: url.toString(), name: "queue-metrics-simulator" });
 }
@@ -419,8 +421,18 @@ async function main() {
     process.exit(1);
   }
   const bucketSec = Number(flags.bucket ?? 10);
+  if (!Number.isFinite(bucketSec) || bucketSec <= 0) {
+    console.error(`--bucket must be a positive number of seconds, got: ${flags.bucket}`);
+    process.exit(1);
+  }
   const windowSec = parseDuration(flags.window ?? "2h");
   const totalBuckets = Math.floor(windowSec / bucketSec);
+  if (!Number.isFinite(totalBuckets) || totalBuckets <= 0) {
+    console.error(
+      `--window must be longer than --bucket (got ${windowSec}s window, ${bucketSec}s bucket)`
+    );
+    process.exit(1);
+  }
   const seed = Number(flags.seed ?? 1);
   const live = flags.live === "true";