|
| 1 | +import { describe, expect, it } from "vitest"; |
| 2 | +import * as fc from "fast-check"; |
| 3 | +import { withQueuedRetry } from "../../lib/codex-manager/settings-write-queue.js"; |
| 4 | + |
| 5 | +type TaskBehavior = "ok" | "flaky" | "fatal"; |
| 6 | + |
| 7 | +const arbSchedule = fc.array( |
| 8 | + fc.record({ |
| 9 | + key: fc.integer({ min: 0, max: 2 }), |
| 10 | + behavior: fc.constantFrom<TaskBehavior>("ok", "flaky", "fatal"), |
| 11 | + }), |
| 12 | + { minLength: 1, maxLength: 12 }, |
| 13 | +); |
| 14 | + |
| 15 | +function errnoError(code: string): NodeJS.ErrnoException { |
| 16 | + return Object.assign(new Error(code), { code }); |
| 17 | +} |
| 18 | + |
| 19 | +const immediateSleep = { sleep: async () => {} }; |
| 20 | + |
| 21 | +// Unique key namespace per run so the module-level queue map never couples |
| 22 | +// property iterations. |
| 23 | +let runCounter = 0; |
| 24 | + |
| 25 | +describe("withQueuedRetry serialization properties", () => { |
| 26 | + it("keeps every key's tasks contiguous and in submission order, for any schedule", async () => { |
| 27 | + await fc.assert( |
| 28 | + fc.asyncProperty(arbSchedule, async (schedule) => { |
| 29 | + const run = runCounter++; |
| 30 | + // Invocation log per key: the task index for every task() call, |
| 31 | + // including retries. |
| 32 | + const invocationsByKey = new Map<number, number[]>(); |
| 33 | + const flakyFailed = new Set<number>(); |
| 34 | + |
| 35 | + const pending = schedule.map((spec, taskIndex) => |
| 36 | + withQueuedRetry( |
| 37 | + `/settings/prop-${run}-key-${spec.key}.json`, |
| 38 | + async () => { |
| 39 | + const log = invocationsByKey.get(spec.key) ?? []; |
| 40 | + log.push(taskIndex); |
| 41 | + invocationsByKey.set(spec.key, log); |
| 42 | + if (spec.behavior === "fatal") { |
| 43 | + throw errnoError("ENOSPC"); |
| 44 | + } |
| 45 | + if (spec.behavior === "flaky" && !flakyFailed.has(taskIndex)) { |
| 46 | + flakyFailed.add(taskIndex); |
| 47 | + throw errnoError("EBUSY"); |
| 48 | + } |
| 49 | + return `result-${taskIndex}`; |
| 50 | + }, |
| 51 | + immediateSleep, |
| 52 | + ).then( |
| 53 | + (value) => ({ taskIndex, outcome: "ok" as const, value }), |
| 54 | + () => ({ taskIndex, outcome: "error" as const }), |
| 55 | + ), |
| 56 | + ); |
| 57 | + const settled = await Promise.all(pending); |
| 58 | + |
| 59 | + // Outcomes match behaviors: fatal rejects, ok/flaky resolve with |
| 60 | + // their own result; a failed predecessor never blocks successors. |
| 61 | + for (const result of settled) { |
| 62 | + const spec = schedule[result.taskIndex]; |
| 63 | + if (spec.behavior === "fatal") { |
| 64 | + expect(result.outcome).toBe("error"); |
| 65 | + } else { |
| 66 | + expect(result).toMatchObject({ |
| 67 | + outcome: "ok", |
| 68 | + value: `result-${result.taskIndex}`, |
| 69 | + }); |
| 70 | + } |
| 71 | + } |
| 72 | + |
| 73 | + // Per key: invocations form contiguous groups (retries never |
| 74 | + // interleave with another task) and groups run in submission order. |
| 75 | + for (const [, invocations] of invocationsByKey) { |
| 76 | + const groups: number[] = []; |
| 77 | + for (const taskIndex of invocations) { |
| 78 | + if (groups[groups.length - 1] !== taskIndex) { |
| 79 | + groups.push(taskIndex); |
| 80 | + } |
| 81 | + } |
| 82 | + expect(new Set(groups).size).toBe(groups.length); |
| 83 | + expect([...groups].sort((a, b) => a - b)).toEqual(groups); |
| 84 | + } |
| 85 | + |
| 86 | + // Every task ran at least once, on its own key. |
| 87 | + const totalInvocations = [...invocationsByKey.values()].flat(); |
| 88 | + for (let taskIndex = 0; taskIndex < schedule.length; taskIndex += 1) { |
| 89 | + expect(totalInvocations).toContain(taskIndex); |
| 90 | + expect( |
| 91 | + invocationsByKey.get(schedule[taskIndex].key) ?? [], |
| 92 | + ).toContain(taskIndex); |
| 93 | + } |
| 94 | + }), |
| 95 | + ); |
| 96 | + }); |
| 97 | + |
| 98 | + it("clamps any 429 retry-after hint into the 10ms..30s range", async () => { |
| 99 | + await fc.assert( |
| 100 | + fc.asyncProperty( |
| 101 | + fc.integer({ min: 1, max: 2_000_000_000 }), |
| 102 | + async (retryAfterMs) => { |
| 103 | + const run = runCounter++; |
| 104 | + const delays: number[] = []; |
| 105 | + let failed = false; |
| 106 | + |
| 107 | + await withQueuedRetry( |
| 108 | + `/settings/prop-${run}-clamp.json`, |
| 109 | + async () => { |
| 110 | + if (!failed) { |
| 111 | + failed = true; |
| 112 | + throw Object.assign(new Error("throttled"), { |
| 113 | + status: 429, |
| 114 | + retryAfterMs, |
| 115 | + }); |
| 116 | + } |
| 117 | + return "written"; |
| 118 | + }, |
| 119 | + { |
| 120 | + sleep: async (ms: number) => { |
| 121 | + delays.push(ms); |
| 122 | + }, |
| 123 | + }, |
| 124 | + ); |
| 125 | + |
| 126 | + expect(delays).toEqual([ |
| 127 | + Math.max(10, Math.min(30_000, Math.round(retryAfterMs))), |
| 128 | + ]); |
| 129 | + }, |
| 130 | + ), |
| 131 | + ); |
| 132 | + }); |
| 133 | +}); |
0 commit comments