fix(metrics-pipeline): widen order_key packing factor to 1e6

Packs the stream sequence with a 1e6 factor (was 1e5) so up to 1M entries per millisecond per shard fit before a seq could spill into the next millisecond's range, far above what a single Redis stream can produce. ms*1e6 stays within UInt64. Also fixes the webapp mapping test that still expected a numeric order_key after the switch to a BigInt-derived string.

Author: ericallam

apps/webapp/test/queueMetricsMapping.test.ts

@@ -112,7 +112,7 @@ describe("mapEntryToRow", () => {
         op: "started",
         wait_ms: 48,
         cumulative: 512,
-        order_key: 1700000000000 * 100000,
+        order_key: (1700000000000n * 1000000n).toString(),
       })
     );
     expect(mapEntryToRow({ id: "1-0", fields: { op: "ack", q, cum: "9" } })).toEqual(

internal-packages/clickhouse/schema/035_create_queue_metrics_v1.sql

@@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS trigger_dev.queue_metrics_raw_v1
   environment_id   String CODEC(ZSTD(1)),
   queue_name       String CODEC(ZSTD(1)),
   event_time       DateTime CODEC(Delta(4), ZSTD(1)),
-  order_key        UInt64 DEFAULT 0,                 -- stream-id composite (ms*1e5+seq), deltaSumTimestamp ordering key
+  order_key        UInt64 DEFAULT 0,                 -- stream-id composite (ms*1e6+seq), deltaSumTimestamp ordering key
   op               LowCardinality(String),          -- gauge | enqueue | started | ack | nack | dlq
   running          UInt32 DEFAULT 0,
   queued           UInt32 DEFAULT 0,

internal-packages/metrics-pipeline/src/pipeline.test.ts

@@ -35,14 +35,14 @@ describe("stream keys", () => {
   });
 
   it("entryOrderKey stays exact and strictly monotonic at real epoch magnitudes", () => {
-    const ms = 1783000000000; // ~2026: ms*1e5 is past JS safe-integer range, so a number key
+    const ms = 1783000000000; // ~2026: ms*1e6 is past JS safe-integer range, so a number key
     const k = (seq: number) => BigInt(entryOrderKey(`${ms}-${seq}`));
     // adjacent seq within one ms must not collapse to the same key (the float bug)
-    expect(k(0)).toBe(BigInt(ms) * 100000n);
+    expect(k(0)).toBe(BigInt(ms) * 1000000n);
     expect(k(1) - k(0)).toBe(1n);
     expect(k(2) - k(1)).toBe(1n);
-    // a later ms always outranks any seq of an earlier ms
-    expect(BigInt(entryOrderKey(`${ms + 1}-0`))).toBeGreaterThan(k(99999));
+    // a later ms always outranks any seq of an earlier ms (up to the 1M/ms factor)
+    expect(BigInt(entryOrderKey(`${ms + 1}-0`))).toBeGreaterThan(k(999999));
   });
 });
 

internal-packages/metrics-pipeline/src/types.ts

@@ -32,10 +32,11 @@ export function entryTimeMs(id: string): number | null {
   return Number.isFinite(ms) ? ms : null;
 }
 
-// Ordering key from a stream id (`<ms>-<seq>`) = ms*1e5+seq, for deltaSumTimestamp. BigInt +
-// string because ms*1e5 exceeds JS safe-integer range at real epoch magnitudes (a number would
+// Ordering key from a stream id (`<ms>-<seq>`) = ms*1e6+seq, for deltaSumTimestamp. BigInt +
+// string because ms*1e6 exceeds JS safe-integer range at real epoch magnitudes (a number would
 // collapse nearby seq values); the ClickHouse order_key column is UInt64 and takes the string.
+// The 1e6 factor (1M entries/ms/shard, far above any single Redis stream) stays within UInt64.
 export function entryOrderKey(id: string): string {
   const [ms, seq] = id.split("-");
-  return (BigInt(Number(ms) || 0) * 100000n + BigInt(Number(seq) || 0)).toString();
+  return (BigInt(Number(ms) || 0) * 1000000n + BigInt(Number(seq) || 0)).toString();
 }