Merge branch 'main' into christiaan/minimax-free

Author: chrarnoldus

cloudflare-gastown/src/dos/Town.do.ts

@@ -240,7 +240,11 @@ export class TownDO extends DurableObject<Env> {
   }
 
   private emitEvent(data: Omit<GastownEventData, 'userId' | 'delivery'>): void {
-    writeEvent(this.env, { ...data, delivery: 'internal', userId: this._ownerUserId });
+    writeEvent(this.env, {
+      ...data,
+      delivery: 'internal',
+      userId: this._ownerUserId,
+    });
   }
 
   /** Build the context object used by the scheduling sub-module. */
@@ -290,7 +294,9 @@ export class TownDO extends DurableObject<Env> {
           });
         }
 
-        return scheduling.dispatchAgent(schedulingCtx, agent, bead, { systemPromptOverride });
+        return scheduling.dispatchAgent(schedulingCtx, agent, bead, {
+          systemPromptOverride,
+        });
       },
       stopAgent: async agentId => {
         await dispatch.stopAgentInContainer(this.env, this.townId, agentId);
@@ -677,7 +683,9 @@ export class TownDO extends DurableObject<Env> {
       const townConfig = await this.getTownConfig();
       if (!townConfig.kilocode_token || townConfig.kilocode_token !== rigConfig.kilocodeToken) {
         console.log(`${TOWN_LOG} configureRig: propagating kilocodeToken to town config`);
-        await this.updateTownConfig({ kilocode_token: rigConfig.kilocodeToken });
+        await this.updateTownConfig({
+          kilocode_token: rigConfig.kilocodeToken,
+        });
       }
     }
 
@@ -1214,10 +1222,14 @@ export class TownDO extends DurableObject<Env> {
    * Return undelivered, non-expired nudges for an agent.
    * Urgent nudges are returned first, then FIFO within same priority.
    */
-  async getPendingNudges(
-    agentId: string
-  ): Promise<
-    { nudge_id: string; message: string; mode: string; priority: string; source: string }[]
+  async getPendingNudges(agentId: string): Promise<
+    {
+      nudge_id: string;
+      message: string;
+      mode: string;
+      priority: string;
+      source: string;
+    }[]
   > {
     const rows = [
       ...query(
@@ -1722,6 +1734,14 @@ export class TownDO extends DurableObject<Env> {
     return reviewQueue.advanceMoleculeStep(this.sql, agentId, summary);
   }
 
+  async getMergeQueueData(params: {
+    rigId?: string;
+    limit?: number;
+    since?: string;
+  }): Promise<reviewQueue.MergeQueueData> {
+    return reviewQueue.getMergeQueueData(this.sql, params);
+  }
+
   // ══════════════════════════════════════════════════════════════════
   // Atomic Sling (create bead + agent + hook)
   // ══════════════════════════════════════════════════════════════════
@@ -1768,7 +1788,12 @@ export class TownDO extends DurableObject<Env> {
 
   /** Build the rig list for mayor agent startup (browse worktree setup on fresh containers). */
   private async rigListForMayor(): Promise<
-    Array<{ rigId: string; gitUrl: string; defaultBranch: string; platformIntegrationId?: string }>
+    Array<{
+      rigId: string;
+      gitUrl: string;
+      defaultBranch: string;
+      platformIntegrationId?: string;
+    }>
   > {
     const rigRecords = rigs.listRigs(this.sql);
     return Promise.all(
@@ -1792,7 +1817,10 @@ export class TownDO extends DurableObject<Env> {
     message: string,
     _model?: string,
     uiContext?: string
-  ): Promise<{ agentId: string; sessionStatus: 'idle' | 'active' | 'starting' }> {
+  ): Promise<{
+    agentId: string;
+    sessionStatus: 'idle' | 'active' | 'starting';
+  }> {
     const townId = this.townId;
 
     let mayor = agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null;
@@ -1876,7 +1904,10 @@ export class TownDO extends DurableObject<Env> {
    * Called eagerly on page load so the terminal is available immediately
    * without requiring the user to send a message first.
    */
-  async ensureMayor(): Promise<{ agentId: string; sessionStatus: 'idle' | 'active' | 'starting' }> {
+  async ensureMayor(): Promise<{
+    agentId: string;
+    sessionStatus: 'idle' | 'active' | 'starting';
+  }> {
     const townId = this.townId;
 
     let mayor = agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null;
@@ -2250,7 +2281,10 @@ export class TownDO extends DurableObject<Env> {
     tasks: Array<{ title: string; body?: string; depends_on?: number[] }>;
     merge_mode?: 'review-then-land' | 'review-and-merge';
     staged?: boolean;
-  }): Promise<{ convoy: ConvoyEntry; beads: Array<{ bead: Bead; agent: Agent | null }> }> {
+  }): Promise<{
+    convoy: ConvoyEntry;
+    beads: Array<{ bead: Bead; agent: Agent | null }>;
+  }> {
     // Resolve staged: explicit request wins, otherwise fall back to town config default.
     const townConfig = await this.getTownConfig();
     const isStaged = input.staged ?? townConfig.staged_convoys_default;
@@ -2449,9 +2483,10 @@ export class TownDO extends DurableObject<Env> {
   /**
    * Transition a staged convoy to active: hook agents and begin dispatch.
    */
-  async startConvoy(
-    convoyId: string
-  ): Promise<{ convoy: ConvoyEntry; beads: Array<{ bead: Bead; agent: Agent | null }> }> {
+  async startConvoy(convoyId: string): Promise<{
+    convoy: ConvoyEntry;
+    beads: Array<{ bead: Bead; agent: Agent | null }>;
+  }> {
     const convoy = this.getConvoy(convoyId);
     if (!convoy) throw new Error(`Convoy not found: ${convoyId}`);
     if (!convoy.staged) throw new Error(`Convoy is not staged: ${convoyId}`);
@@ -2994,9 +3029,14 @@ export class TownDO extends DurableObject<Env> {
       const violations = reconciler.checkInvariants(this.sql);
       metrics.invariantViolations = violations.length;
       if (violations.length > 0) {
-        console.error(
-          `${TOWN_LOG} [reconciler:invariants] town=${townId} ${violations.length} violation(s): ${JSON.stringify(violations)}`
-        );
+        // Emit as an analytics event for observability dashboards instead
+        // of console.error (which spams Workers logs every 5s per town).
+        this.emitEvent({
+          event: 'reconciler.invariant_violations',
+          townId,
+          label: violations.map(v => `[${v.invariant}] ${v.message}`).join('; '),
+          value: violations.length,
+        });
       }
     } catch (err) {
       console.warn(`${TOWN_LOG} [reconciler:invariants] town=${townId} check failed`, err);
@@ -3575,7 +3615,13 @@ export class TownDO extends DurableObject<Env> {
         []
       ),
     ];
-    const beadCounts = { open: 0, inProgress: 0, inReview: 0, failed: 0, triageRequests: 0 };
+    const beadCounts = {
+      open: 0,
+      inProgress: 0,
+      inReview: 0,
+      failed: 0,
+      triageRequests: 0,
+    };
     for (const row of beadRows) {
       const s = `${row.status as string}`;
       const c = Number(row.cnt);

cloudflare-gastown/src/dos/town/actions.ts

@@ -576,7 +576,11 @@ export function applyAction(ctx: ApplyActionContext, action: Action): (() => Pro
     }
 
     case 'send_nudge': {
-      // Insert nudge record synchronously
+      // Insert nudge record synchronously.
+      // Explicitly set created_at to ISO 8601 so it matches the format used
+      // by hasRecentNudge's cutoff comparison (#1412). SQLite's default
+      // datetime('now') produces 'YYYY-MM-DD HH:MM:SS' (space separator)
+      // which compares incorrectly against JS toISOString().
       const nudgeId = crypto.randomUUID();
       query(
         sql,
@@ -588,10 +592,18 @@ export function applyAction(ctx: ApplyActionContext, action: Action): (() => Pro
             ${agent_nudges.columns.mode},
             ${agent_nudges.columns.priority},
             ${agent_nudges.columns.source},
+            ${agent_nudges.columns.created_at},
             ${agent_nudges.columns.expires_at}
-          ) VALUES (?, ?, ?, 'immediate', 'urgent', ?, ?)
+          ) VALUES (?, ?, ?, 'immediate', 'urgent', ?, ?, ?)
         `,
-        [nudgeId, action.agent_id, action.message, `reconciler:${action.tier}`, null]
+        [
+          nudgeId,
+          action.agent_id,
+          action.message,
+          `reconciler:${action.tier}`,
+          new Date().toISOString(),
+          null,
+        ]
       );
 
       return async () => {

cloudflare-gastown/src/dos/town/agents.ts

@@ -546,11 +546,20 @@ export function touchAgent(
     activeTools?: string[];
   }
 ): void {
+  // A heartbeat is proof the agent is alive in the container.
+  // If the agent's status is 'idle' (e.g. due to a dispatch timeout
+  // race — see #1358), restore it to 'working'. This prevents the
+  // reconciler from treating the agent as lost while it's actively
+  // sending heartbeats.
   query(
     sql,
     /* sql */ `
       UPDATE ${agent_metadata}
       SET ${agent_metadata.columns.last_activity_at} = ?,
+          ${agent_metadata.columns.status} = CASE
+            WHEN ${agent_metadata.columns.status} = 'idle' THEN 'working'
+            ELSE ${agent_metadata.columns.status}
+          END,
           ${agent_metadata.columns.last_event_type} = COALESCE(?, ${agent_metadata.columns.last_event_type}),
           ${agent_metadata.columns.last_event_at} = COALESCE(?, ${agent_metadata.columns.last_event_at}),
           ${agent_metadata.columns.active_tools} = COALESCE(?, ${agent_metadata.columns.active_tools})

cloudflare-gastown/src/dos/town/reconciler.ts

@@ -249,7 +249,18 @@ export function applyEvent(sql: SqlStorage, event: TownEventRecord): void {
       const agent = agents.getAgent(sql, event.agent_id);
       if (!agent) return;
 
-      // Only act on working/stalled agents whose container has stopped
+      // Only act on working/stalled agents whose container has stopped.
+      // For 'not_found': skip if the agent was dispatched recently (#1358).
+      // During a cold start the container may 404 on /agents/:id/status
+      // because the agent hasn't registered in the process manager yet.
+      // The 3-minute grace period covers the 60s HTTP timeout plus
+      // typical cold start time (git clone + worktree). Truly dead
+      // agents are caught by reconcileAgents after 90s of no heartbeats.
+      if (containerStatus === 'not_found' && agent.last_activity_at) {
+        const ageSec = (Date.now() - new Date(agent.last_activity_at).getTime()) / 1000;
+        if (ageSec < 180) return; // 3-minute grace for cold starts
+      }
+
       if (
         (agent.status === 'working' || agent.status === 'stalled') &&
         (containerStatus === 'exited' || containerStatus === 'not_found')
@@ -340,6 +351,9 @@ export function reconcileAgents(sql: SqlStorage): Action[] {
   ]);
 
   for (const agent of workingAgents) {
+    // Mayors are always working with no hook — skip them
+    if (agent.role === 'mayor') continue;
+
     if (!agent.last_activity_at) {
       // No heartbeat ever received — container may have failed to start
       actions.push({
@@ -357,6 +371,18 @@ export function reconcileAgents(sql: SqlStorage): Action[] {
         to: 'idle',
         reason: 'heartbeat lost (3 missed cycles)',
       });
+    } else if (!agent.current_hook_bead_id) {
+      // Agent is working with fresh heartbeat but no hook — it's running
+      // in the container but has no bead to work on (gt_done already ran,
+      // or the hook was cleared by another code path). Set to idle so
+      // processReviewQueue / schedulePendingWork can use it.
+      actions.push({
+        type: 'transition_agent',
+        agent_id: agent.bead_id,
+        from: 'working',
+        to: 'idle',
+        reason: 'working agent has no hook (gt_done already completed)',
+      });
     }
   }
 
@@ -599,18 +625,24 @@ export function reconcileBeads(sql: SqlStorage): Action[] {
   for (const bead of staleInProgress) {
     if (!staleMs(bead.updated_at, STALE_IN_PROGRESS_TIMEOUT_MS)) continue;
 
-    // Check if any agent is hooked AND working/stalled
+    // Check if any agent is hooked AND (working/stalled OR has a recent
+    // heartbeat). The heartbeat check is defense-in-depth for #1358: if
+    // the agent's status is wrong (e.g. stuck on 'idle' due to a dispatch
+    // timeout race), a fresh heartbeat proves the agent is alive.
     const hookedAgent = z
-      .object({ status: z.string() })
+      .object({ status: z.string(), last_activity_at: z.string().nullable() })
       .array()
       .parse([
         ...query(
           sql,
           /* sql */ `
-          SELECT ${agent_metadata.status}
+          SELECT ${agent_metadata.status}, ${agent_metadata.last_activity_at}
           FROM ${agent_metadata}
           WHERE ${agent_metadata.current_hook_bead_id} = ?
-            AND ${agent_metadata.status} IN ('working', 'stalled')
+            AND (
+              ${agent_metadata.status} IN ('working', 'stalled')
+              OR ${agent_metadata.last_activity_at} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-90 seconds')
+            )
         `,
           [bead.bead_id]
         ),
@@ -991,7 +1023,11 @@ export function reconcileReviewQueue(sql: SqlStorage): Action[] {
     }
 
     const mrRows = z
-      .object({ status: z.string(), type: z.string(), rig_id: z.string().nullable() })
+      .object({
+        status: z.string(),
+        type: z.string(),
+        rig_id: z.string().nullable(),
+      })
       .array()
       .parse([
         ...query(
@@ -1231,6 +1267,7 @@ export function reconcileGUPP(sql: SqlStorage): Action[] {
         FROM ${agent_metadata}
         LEFT JOIN ${beads} b ON b.${beads.columns.bead_id} = ${agent_metadata.bead_id}
         WHERE ${agent_metadata.status} IN ('working', 'stalled')
+          AND ${agent_metadata.role} != 'mayor'
       `,
       []
     ),
@@ -1420,18 +1457,20 @@ function getIdleAgentHookedTo(sql: SqlStorage, beadId: string): string | null {
 function hasRecentNudge(sql: SqlStorage, agentId: string, tier: string): boolean {
   // Check if a nudge with this exact tier source was created in the last 60 min.
   // The source is set to `reconciler:${tier}` by applyAction('send_nudge').
-  const cutoff = new Date(Date.now() - 60 * 60_000).toISOString();
+  // Use SQLite's datetime() for the cutoff so the comparison works regardless
+  // of whether created_at was stored in SQLite's native 'YYYY-MM-DD HH:MM:SS'
+  // format (old rows) or ISO 8601 'YYYY-MM-DDTHH:MM:SS.000Z' (new rows).
   const rows = [
     ...query(
       sql,
       /* sql */ `
         SELECT 1 FROM ${agent_nudges}
         WHERE ${agent_nudges.agent_bead_id} = ?
           AND ${agent_nudges.source} = ?
-          AND ${agent_nudges.created_at} > ?
+          AND datetime(${agent_nudges.created_at}) > datetime('now', '-60 minutes')
         LIMIT 1
       `,
-      [agentId, `reconciler:${tier}`, cutoff]
+      [agentId, `reconciler:${tier}`]
     ),
   ];
   return rows.length > 0;