fix(kiloclaw): omit ord from Fly region list due to provisioning issues (#1090)

* fix(kiloclaw): omit ord from Fly region list due to provisioning issues

* revert(kiloclaw): restore shuffle+deprioritize in capacity recovery (#1059)

With explicit region codes instead of the 'us' alias, deprioritizeRegion
can now correctly match concrete regions (e.g. 'iad') against the list.
The original bug — meta-region 'us' never matching concrete region codes —
no longer applies.

Author: jrf0110

kiloclaw/src/config.ts

@@ -38,8 +38,9 @@ export const DEFAULT_MACHINE_GUEST = {
 export const DEFAULT_VOLUME_SIZE_GB = 10;
 
 /** Default Fly region priority list when FLY_REGION env var is not set.
- *  Callers shuffle before selecting so order here doesn't matter. */
-export const DEFAULT_FLY_REGION = 'us,eu';
+ *  Callers shuffle before selecting so order here doesn't matter.
+ *  ord omitted due to provisioning issues. */
+export const DEFAULT_FLY_REGION = 'dfw,ewr,iad,lax,sjc,eu';
 
 // Alarm cadence by instance status
 /** Running machines: fast health checks */

kiloclaw/src/durable-objects/kiloclaw-instance.test.ts

@@ -156,7 +156,7 @@ function createFakeEnv() {
   return {
     FLY_API_TOKEN: 'test-token',
     FLY_APP_NAME: 'test-app',
-    FLY_REGION: 'us,eu',
+    FLY_REGION: 'dfw,ewr,iad,lax,sjc,eu',
     GATEWAY_TOKEN_SECRET: 'test-secret',
     NEXTAUTH_SECRET: 'test-nextauth-secret-at-least-32-chars',
     WORKER_ENV: 'development',
@@ -2128,7 +2128,7 @@ describe('start: volume region validation', () => {
 // ============================================================================
 
 describe('start: 412 insufficient resources recovery', () => {
-  it('fresh provision (never started): deletes volume and creates fresh with reversed regions', async () => {
+  it('fresh provision (never started): deletes volume and creates fresh with deprioritized regions', async () => {
     const { instance, storage } = createInstance();
     await seedProvisioned(storage, { flyMachineId: null, lastStartedAt: null });
 
@@ -2150,15 +2150,23 @@ describe('start: 412 insufficient resources recovery', () => {
 
     // Old volume was deleted
     expect(flyClient.deleteVolume).toHaveBeenCalledWith(expect.anything(), 'vol-1');
-    // New volume created via fallback with reversed regions and compute hint
+    // New volume created via fallback with deprioritized regions and compute hint
     const regions412Call = (flyClient.createVolumeWithFallback as Mock).mock.calls[0];
     expect(regions412Call[1]).toEqual(
       expect.objectContaining({
         compute: expect.objectContaining({ cpus: 2, memory_mb: 3072 }) as unknown,
       })
     );
-    // Regions are reversed from FLY_REGION='us,eu' so EU is tried first on recovery
-    expect(regions412Call[2]).toEqual(['eu', 'us']);
+    // Regions are shuffled, so just check the set (deprioritize is a no-op here
+    // because 'iad' is not in FLY_REGION='dfw,ewr,iad,lax,sjc,eu')
+    expect((regions412Call[2] as string[]).sort()).toEqual([
+      'dfw',
+      'eu',
+      'ewr',
+      'iad',
+      'lax',
+      'sjc',
+    ]);
     // source_volume_id should NOT be set for fresh provision
     const createVolumeCall = (flyClient.createVolumeWithFallback as Mock).mock
       .calls[0][1] as Record<string, unknown>;
@@ -2196,16 +2204,23 @@ describe('start: 412 insufficient resources recovery', () => {
 
     await instance.start('user-1');
 
-    // Volume was forked (source_volume_id set) with compute hint and reversed regions
+    // Volume was forked (source_volume_id set) with compute hint and deprioritized regions
     const regionsForkCall = (flyClient.createVolumeWithFallback as Mock).mock.calls[0];
     expect(regionsForkCall[1]).toEqual(
       expect.objectContaining({
         source_volume_id: 'vol-1',
         compute: expect.objectContaining({ cpus: 2, memory_mb: 3072 }) as unknown,
       })
     );
-    // Regions are reversed from FLY_REGION='us,eu' so EU is tried first on recovery
-    expect(regionsForkCall[2]).toEqual(['eu', 'us']);
+    // Regions are shuffled — check the set
+    expect((regionsForkCall[2] as string[]).sort()).toEqual([
+      'dfw',
+      'eu',
+      'ewr',
+      'iad',
+      'lax',
+      'sjc',
+    ]);
     // Old volume was deleted
     expect(flyClient.deleteVolume).toHaveBeenCalledWith(expect.anything(), 'vol-1');
     // Machine was retried
@@ -2269,8 +2284,15 @@ describe('start: 412 insufficient resources recovery', () => {
         compute: expect.objectContaining({ cpus: 2, memory_mb: 3072 }) as unknown,
       })
     );
-    // Regions are reversed from FLY_REGION='us,eu' so EU is tried first on recovery
-    expect(regionsUpdateCall[2]).toEqual(['eu', 'us']);
+    // Regions are shuffled then deprioritized — check the set
+    expect((regionsUpdateCall[2] as string[]).sort()).toEqual([
+      'dfw',
+      'eu',
+      'ewr',
+      'iad',
+      'lax',
+      'sjc',
+    ]);
     // New machine was created
     expect(storage._store.get('flyMachineId')).toBe('machine-new');
     expect(storage._store.get('flyVolumeId')).toBe('vol-new');

kiloclaw/src/durable-objects/kiloclaw-instance/fly-machines.ts

@@ -8,7 +8,7 @@ import {
   DEFAULT_FLY_REGION,
   STALE_PROVISION_THRESHOLD_MS,
 } from '../../config';
-import { parseRegions, shuffleRegions } from '../regions';
+import { parseRegions, shuffleRegions, deprioritizeRegion } from '../regions';
 import { guestFromSize, volumeNameFromSandboxId } from '../machine-config';
 import type { InstanceMutableState } from './types';
 import { storageUpdate } from './state';
@@ -65,7 +65,8 @@ export async function replaceStrandedVolume(
   const oldVolumeId = state.flyVolumeId;
   const oldRegion = state.flyRegion;
   const hasUserData = state.lastStartedAt !== null;
-  const regions = parseRegions(env.FLY_REGION ?? DEFAULT_FLY_REGION).reverse();
+  const allRegions = shuffleRegions(parseRegions(env.FLY_REGION ?? DEFAULT_FLY_REGION));
+  const regions = deprioritizeRegion(allRegions, oldRegion);
   const compute = guestFromSize(state.machineSize);
 
   // Destroy existing machine if any — it's stuck on the constrained host.

kiloclaw/wrangler.jsonc

@@ -70,7 +70,7 @@
     "FLY_REGISTRY_APP": "kiloclaw-machines", // Shared Docker image registry
     "FLY_ORG_SLUG": "kilo-679", // Org for creating per-user Fly apps
     "FLY_IMAGE_TAG": "latest",
-    "FLY_REGION": "us,eu",
+    "FLY_REGION": "dfw,ewr,iad,lax,sjc,eu",
     "OPENCLAW_ALLOWED_ORIGINS": "https://claw.kilosessions.ai,https://kilo.ai,https://www.kilo.ai",
     "REQUIRE_PROXY_TOKEN": "true",
     // Defaults to "production". For local dev, override to "development" in .dev.vars.