@@ -36,6 +36,7 @@ import {
3636import { CODEX_UNKNOWN_USAGE_SCORE } from "../src/codex/quota" ;
3737import { MAIN_CODEX_ACCOUNT_ID } from "../src/codex/main-account" ;
3838import { routeModel } from "../src/router" ;
39+ import { consumeForInspection } from "../src/server/relay" ;
3940import type { OcxConfig } from "../src/types" ;
4041
4142const TEST_DIR = join ( import . meta. dir , ".tmp-codex-routing-test" ) ;
@@ -65,6 +66,12 @@ function saveTestCredential(id: string): void {
6566 } ) ;
6667}
6768
69+ const inspectionTick = ( ) => new Promise ( resolve => setTimeout ( resolve , 5 ) ) ;
70+
71+ function pendingInspectionStream ( ) : ReadableStream < Uint8Array > {
72+ return new ReadableStream < Uint8Array > ( { start ( ) { } , pull ( ) { } } ) ;
73+ }
74+
6875describe ( "codex routing" , ( ) => {
6976 beforeEach ( ( ) => {
7077 previousOpencodexHome = process . env . OPENCODEX_HOME ;
@@ -339,10 +346,9 @@ describe("codex routing", () => {
339346 recordCodexUpstreamOutcome ( config , "a" , 200 , { now : now + 1 } ) ;
340347 recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 2 } ) ;
341348 recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 3 } ) ;
342- // 2 failures after a success (streak=2) is under the failover threshold (3),
343- // but soft-avoid still blocks for 30s. After it expires, "a" is selectable.
344- // Soft-avoid was last set at now+3, so it expires at now+3+30s.
345- const afterSoftAvoid = now + 3 + CODEX_TRANSIENT_SOFT_AVOID_MS + 1 ;
349+ // The success reset the old streak, so the next two failures form escalation
350+ // level 2 (still below failover threshold 3) and avoid the account for 2m.
351+ const afterSoftAvoid = now + 3 + 2 * 60_000 + 1 ;
346352 expect ( resolveCodexAccountForThread ( "next" , config , afterSoftAvoid ) ) . toBe ( "a" ) ;
347353 } ) ;
348354
@@ -356,6 +362,95 @@ describe("codex routing", () => {
356362 expect ( resolveCodexAccountForThread ( "next" , config ) ) . toBe ( "a" ) ;
357363 } ) ;
358364
365+ test ( "inspection client cancellation records no terminal outcome or account penalty" , async ( ) => {
366+ const config = makeConfig ( ) ;
367+ const record = ( status : "completed" | "failed" | "incomplete" , override ?: number ) => {
368+ recordCodexUpstreamOutcome ( config , "a" , status === "failed" ? ( override ?? 502 ) : 200 ) ;
369+ } ;
370+
371+ const preAborted = new AbortController ( ) ;
372+ preAborted . abort ( ) ;
373+ consumeForInspection ( pendingInspectionStream ( ) , record , preAborted . signal ) ;
374+ expect ( getCodexUpstreamHealth ( "a" ) ) . toBeNull ( ) ;
375+
376+ const midDrain = new AbortController ( ) ;
377+ consumeForInspection ( pendingInspectionStream ( ) , record , midDrain . signal ) ;
378+ midDrain . abort ( ) ;
379+ await inspectionTick ( ) ;
380+ expect ( getCodexUpstreamHealth ( "a" ) ) . toBeNull ( ) ;
381+ } ) ;
382+
383+ test ( "inspection read rejection reports failed plus synthetic 502 and clears affinity" , async ( ) => {
384+ const config = makeConfig ( ) ;
385+ const now = 1_800_000_000_000 ;
386+ updateAccountQuota ( "a" , 10 ) ;
387+ updateAccountQuota ( "b" , 20 ) ;
388+ expect ( resolveCodexAccountForThread ( "reset-thread" , config , now ) ) . toBe ( "a" ) ;
389+ const terminals : Array < [ string , number | undefined ] > = [ ] ;
390+ const resetStream = new ReadableStream < Uint8Array > ( {
391+ start ( controller ) {
392+ controller . error ( new Error ( "socket reset" ) ) ;
393+ } ,
394+ } ) ;
395+
396+ consumeForInspection ( resetStream , ( status , override ) => {
397+ terminals . push ( [ status , override ] ) ;
398+ recordCodexUpstreamOutcome ( config , "a" , override ?? 200 , { now : now + 1 , threadId : "reset-thread" } ) ;
399+ } ) ;
400+ await inspectionTick ( ) ;
401+
402+ expect ( terminals ) . toEqual ( [ [ "failed" , 502 ] ] ) ;
403+ expect ( getCodexUpstreamHealth ( "a" ) ) . toMatchObject ( { consecutiveFailures : 1 , lastFailureStatus : 502 } ) ;
404+ expect ( resolveCodexAccountForThread ( "reset-thread" , config , now + 2 ) ) . toBe ( "b" ) ;
405+ } ) ;
406+
407+ test ( "inspection clean EOF remains incomplete and success-like" , async ( ) => {
408+ const config = makeConfig ( ) ;
409+ const terminals : Array < [ string , number | undefined ] > = [ ] ;
410+ const cleanEof = new ReadableStream < Uint8Array > ( { start ( controller ) { controller . close ( ) ; } } ) ;
411+
412+ consumeForInspection ( cleanEof , ( status , override ) => {
413+ terminals . push ( [ status , override ] ) ;
414+ recordCodexUpstreamOutcome ( config , "a" , status === "failed" ? ( override ?? 502 ) : 200 ) ;
415+ } ) ;
416+ await inspectionTick ( ) ;
417+
418+ expect ( terminals ) . toEqual ( [ [ "incomplete" , undefined ] ] ) ;
419+ expect ( getCodexUpstreamHealth ( "a" ) ) . toBeNull ( ) ;
420+ } ) ;
421+
422+ test ( "transient cooldown escalates to 2m, 10m, then the 30m cap" , ( ) => {
423+ const config = makeConfig ( ) ;
424+ const now = 1_800_000_000_000 ;
425+
426+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now } ) ;
427+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 1 } ) ;
428+ expect ( getCodexAccountSoftAvoidUntil ( "a" , now + 1 ) ) . toBe ( now + 1 + 2 * 60_000 ) ;
429+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 2 } ) ;
430+ expect ( getCodexAccountSoftAvoidUntil ( "a" , now + 2 ) ) . toBe ( now + 2 + 10 * 60_000 ) ;
431+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 3 } ) ;
432+ expect ( getCodexAccountSoftAvoidUntil ( "a" , now + 3 ) ) . toBe ( now + 3 + 30 * 60_000 ) ;
433+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 4 } ) ;
434+ expect ( getCodexAccountSoftAvoidUntil ( "a" , now + 4 ) ) . toBe ( now + 4 + 30 * 60_000 ) ;
435+ } ) ;
436+
437+ test ( "escalation level 2 requires two consecutive healthy terminals to clear" , ( ) => {
438+ const config = makeConfig ( ) ;
439+ const now = 1_800_000_000_000 ;
440+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now } ) ;
441+ recordCodexUpstreamOutcome ( config , "a" , 503 , { now : now + 1 } ) ;
442+
443+ recordCodexUpstreamOutcome ( config , "a" , 200 , { now : now + 2 } ) ;
444+ expect ( getCodexUpstreamHealth ( "a" ) ) . toMatchObject ( {
445+ consecutiveFailures : 2 ,
446+ consecutiveSuccesses : 1 ,
447+ } ) ;
448+ expect ( isCodexAccountSoftAvoided ( "a" , now + 2 ) ) . toBe ( true ) ;
449+
450+ recordCodexUpstreamOutcome ( config , "a" , 200 , { now : now + 3 } ) ;
451+ expect ( getCodexUpstreamHealth ( "a" ) ) . toBeNull ( ) ;
452+ } ) ;
453+
359454 test ( "stale thread affinity is revalidated before reuse" , ( ) => {
360455 const config = makeConfig ( ) ;
361456 updateAccountQuota ( "a" , 10 ) ;
@@ -690,9 +785,9 @@ describe("codex routing", () => {
690785 const firstAvoid = getCodexAccountSoftAvoidUntil ( "a" , now ) ;
691786 expect ( firstAvoid ) . toBe ( now + CODEX_TRANSIENT_SOFT_AVOID_MS ) ;
692787
693- // A second failure 10s later extends the window from that point.
788+ // A second failure 10s later escalates the window to 2m from that point.
694789 recordCodexUpstreamOutcome ( config , "a" , "timeout" , { now : now + 10_000 } ) ;
695- expect ( getCodexAccountSoftAvoidUntil ( "a" , now + 10_001 ) ) . toBe ( now + 10_000 + CODEX_TRANSIENT_SOFT_AVOID_MS ) ;
790+ expect ( getCodexAccountSoftAvoidUntil ( "a" , now + 10_001 ) ) . toBe ( now + 10_000 + 2 * 60_000 ) ;
696791 } ) ;
697792
698793 test ( "soft-avoid is not applied when failover threshold is 0" , ( ) => {
0 commit comments