Skip to content

Commit 62b00b2

Browse files
committed
fix(task-persistence): guard concurrent history mutations
1 parent ac5c2e0 commit 62b00b2

7 files changed

Lines changed: 426 additions & 310 deletions

File tree

src/__tests__/history-resume-delegation.spec.ts

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -923,6 +923,72 @@ describe("History resume delegation - parent metadata transitions", () => {
923923
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining("[reopenParentFromDelegation] Aborting"))
924924
})
925925

926+
it("reopenParentFromDelegation revalidates the locked parent snapshot before committing completion metadata", async () => {
927+
const logSpy = vi.fn()
928+
const saveTaskMessagesMock = vi.mocked(saveTaskMessages)
929+
const saveApiMessagesMock = vi.mocked(saveApiMessages)
930+
const createTaskWithHistoryItem = vi.fn()
931+
const parentItem = {
932+
id: "parent-locked-guard",
933+
status: "delegated",
934+
awaitingChildId: "child-locked-guard",
935+
childIds: ["child-locked-guard"],
936+
ts: 100,
937+
task: "Parent locked guard",
938+
tokensIn: 0,
939+
tokensOut: 0,
940+
totalCost: 0,
941+
}
942+
const childItem = {
943+
id: "child-locked-guard",
944+
status: "active",
945+
ts: 101,
946+
task: "Child locked guard",
947+
tokensIn: 0,
948+
tokensOut: 0,
949+
totalCost: 0,
950+
}
951+
const atomicUpdatePair = vi.fn(
952+
async (
953+
_firstId: string,
954+
_secondId: string,
955+
firstUpdater: (h: HistoryItem) => HistoryItem,
956+
secondUpdater: (h: HistoryItem) => HistoryItem,
957+
) => {
958+
firstUpdater(childItem as HistoryItem)
959+
secondUpdater({ ...parentItem, status: "active", awaitingChildId: undefined } as HistoryItem)
960+
return []
961+
},
962+
)
963+
964+
const provider = makeProviderStub({
965+
contextProxy: { globalStorageUri: { fsPath: "/tmp" } },
966+
getTaskWithId: vi.fn().mockResolvedValue({ historyItem: parentItem }),
967+
emit: vi.fn(),
968+
log: logSpy,
969+
getCurrentTask: vi.fn(() => null),
970+
removeClineFromStack: vi.fn(),
971+
createTaskWithHistoryItem,
972+
taskHistoryStore: { atomicUpdatePair, get: vi.fn() },
973+
} as any)
974+
vi.mocked(readTaskMessages).mockResolvedValue([])
975+
vi.mocked(readApiMessages).mockResolvedValue([])
976+
977+
await expect(
978+
(ClineProvider.prototype as any).reopenParentFromDelegation.call(provider, {
979+
parentTaskId: "parent-locked-guard",
980+
childTaskId: "child-locked-guard",
981+
completionResultSummary: "should not commit",
982+
}),
983+
).resolves.toBe(false)
984+
985+
expect(atomicUpdatePair).toHaveBeenCalled()
986+
expect(createTaskWithHistoryItem).not.toHaveBeenCalled()
987+
expect(logSpy).toHaveBeenCalledWith(expect.stringContaining("[reopenParentFromDelegation] Aborting"))
988+
expect(saveTaskMessagesMock).toHaveBeenCalled()
989+
expect(saveApiMessagesMock).toHaveBeenCalled()
990+
})
991+
926992
it("serializes delegation transitions and continues after a rejected predecessor", async () => {
927993
const provider = makeProviderStub({} as any) as any
928994
const calls: string[] = []

src/core/task-persistence/TaskHistoryLock.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,9 @@ export class TaskHistoryLock {
7676
update: 10000,
7777
realpath: false,
7878
retries: {
79-
retries: 10,
80-
factor: 2,
81-
minTimeout: 50,
79+
retries: 36,
80+
factor: 1,
81+
minTimeout: 1000,
8282
maxTimeout: 1000,
8383
},
8484
onCompromised: (err) => {

src/core/task-persistence/TaskHistoryStore.ts

Lines changed: 134 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -261,25 +261,27 @@ export class TaskHistoryStore {
261261
* Delete multiple tasks' history items in a batch.
262262
*/
263263
async deleteMany(taskIds: string[]): Promise<void> {
264-
return this.withLock(async () => {
265-
for (const taskId of taskIds) {
266-
this.cache.delete(taskId)
264+
return this.withLock(() => this.deleteManyCore(taskIds))
265+
}
267266

268-
try {
269-
const filePath = await this.getTaskFilePath(taskId)
270-
await fs.unlink(filePath)
271-
} catch {
272-
// File may already be deleted
273-
}
267+
private async deleteManyCore(taskIds: string[]): Promise<void> {
268+
for (const taskId of taskIds) {
269+
this.cache.delete(taskId)
270+
271+
try {
272+
const filePath = await this.getTaskFilePath(taskId)
273+
await fs.unlink(filePath)
274+
} catch {
275+
// File may already be deleted
274276
}
277+
}
275278

276-
this.scheduleIndexWrite()
279+
this.scheduleIndexWrite()
277280

278-
// Call onWrite callback inside the lock for serialized write-through
279-
if (this.onWrite) {
280-
await this.onWrite(this.getAll())
281-
}
282-
})
281+
// Call onWrite callback inside the lock for serialized write-through
282+
if (this.onWrite) {
283+
await this.onWrite(this.getAll())
284+
}
283285
}
284286

285287
// ────────────────────────────── Reconciliation ──────────────────────────────
@@ -292,50 +294,52 @@ export class TaskHistoryStore {
292294
*/
293295
async reconcile(): Promise<void> {
294296
// Run through the write lock to prevent interleaving with upsert/delete
295-
return this.withLock(async () => {
296-
const tasksDir = await this.getTasksDir()
297+
return this.withLock(() => this.reconcileCore())
298+
}
297299

298-
let dirEntries: string[]
299-
try {
300-
dirEntries = await fs.readdir(tasksDir)
301-
} catch {
302-
return // tasks dir doesn't exist yet
303-
}
300+
private async reconcileCore(): Promise<void> {
301+
const tasksDir = await this.getTasksDir()
304302

305-
// Filter out the index file and hidden files
306-
const taskDirNames = dirEntries.filter((name) => !name.startsWith("_") && !name.startsWith("."))
307-
308-
const onDiskIds = new Set(taskDirNames)
309-
const cacheIds = new Set(this.cache.keys())
310-
let changed = false
311-
312-
// Tasks on disk but not in cache: read their history_item.json
313-
for (const taskId of onDiskIds) {
314-
if (!cacheIds.has(taskId)) {
315-
try {
316-
const item = await this.readTaskFile(taskId)
317-
if (item) {
318-
this.cache.set(taskId, item)
319-
changed = true
320-
}
321-
} catch {
322-
// Corrupted or missing file, skip
303+
let dirEntries: string[]
304+
try {
305+
dirEntries = await fs.readdir(tasksDir)
306+
} catch {
307+
return // tasks dir doesn't exist yet
308+
}
309+
310+
// Filter out the index file and hidden files
311+
const taskDirNames = dirEntries.filter((name) => !name.startsWith("_") && !name.startsWith("."))
312+
313+
const onDiskIds = new Set(taskDirNames)
314+
const cacheIds = new Set(this.cache.keys())
315+
let changed = false
316+
317+
// Tasks on disk but not in cache: read their history_item.json
318+
for (const taskId of onDiskIds) {
319+
if (!cacheIds.has(taskId)) {
320+
try {
321+
const item = await this.readTaskFile(taskId)
322+
if (item) {
323+
this.cache.set(taskId, item)
324+
changed = true
323325
}
326+
} catch {
327+
// Corrupted or missing file, skip
324328
}
325329
}
330+
}
326331

327-
// Tasks in cache but not on disk: remove from cache
328-
for (const taskId of cacheIds) {
329-
if (!onDiskIds.has(taskId)) {
330-
this.cache.delete(taskId)
331-
changed = true
332-
}
332+
// Tasks in cache but not on disk: remove from cache
333+
for (const taskId of cacheIds) {
334+
if (!onDiskIds.has(taskId)) {
335+
this.cache.delete(taskId)
336+
changed = true
333337
}
338+
}
334339

335-
if (changed) {
336-
this.scheduleIndexWrite()
337-
}
338-
})
340+
if (changed) {
341+
this.scheduleIndexWrite()
342+
}
339343
}
340344

341345
/**
@@ -359,69 +363,71 @@ export class TaskHistoryStore {
359363
* A parent awaiting an `active`, `interrupted`, or `delegated` child is left as-is — the child is resumable.
360364
*/
361365
private async reconcileDelegationState(): Promise<void> {
362-
return this.withLock(async () => {
363-
let repairsInThisPass: number
364-
do {
365-
repairsInThisPass = 0
366-
// Rebuild the lookup map each pass so repairs from the previous pass
367-
// are visible when evaluating chained delegations.
368-
const byId = new Map(Array.from(this.cache.values()).map((i) => [i.id, i]))
369-
370-
for (const [, item] of byId) {
371-
if (item.status !== "delegated") {
372-
continue
373-
}
366+
return this.withLock(() => this.reconcileDelegationStateCore())
367+
}
374368

375-
if (!item.awaitingChildId) {
376-
await this.upsertCore(
377-
{ ...item, status: "active", awaitingChildId: undefined, delegatedToId: undefined },
378-
{ skipTransitionCheck: true },
379-
)
380-
console.warn(
381-
`[TaskHistoryStore] Reconciled invalid delegation: task ${item.id} → active (no awaitingChildId)`,
382-
)
383-
repairsInThisPass++
384-
continue
385-
}
369+
private async reconcileDelegationStateCore(): Promise<void> {
370+
let repairsInThisPass: number
371+
do {
372+
repairsInThisPass = 0
373+
// Rebuild the lookup map each pass so repairs from the previous pass
374+
// are visible when evaluating chained delegations.
375+
const byId = new Map(Array.from(this.cache.values()).map((i) => [i.id, i]))
376+
377+
for (const [, item] of byId) {
378+
if (item.status !== "delegated") {
379+
continue
380+
}
386381

387-
const child = byId.get(item.awaitingChildId)
388-
389-
if (!child) {
390-
await this.upsertCore(
391-
{
392-
...item,
393-
status: "active",
394-
awaitingChildId: undefined,
395-
delegatedToId: undefined,
396-
},
397-
{ skipTransitionCheck: true },
398-
)
399-
console.warn(
400-
`[TaskHistoryStore] Reconciled orphaned delegation: task ${item.id} → active (child ${item.awaitingChildId} not found)`,
401-
)
402-
repairsInThisPass++
403-
} else if (child.status === "completed") {
404-
await this.upsertCore(
405-
{
406-
...item,
407-
status: "active",
408-
awaitingChildId: undefined,
409-
delegatedToId: undefined,
410-
completedByChildId: child.id,
411-
completionResultSummary:
412-
child.completionResultSummary ?? "Task completed (recovered after interruption)",
413-
},
414-
{ skipTransitionCheck: true },
415-
)
416-
console.warn(
417-
`[TaskHistoryStore] Reconciled interrupted handoff: task ${item.id} → active (child ${item.awaitingChildId} already completed)`,
418-
)
419-
repairsInThisPass++
420-
}
421-
// child.status === "active", "interrupted", or "delegated" → leave as-is this pass
382+
if (!item.awaitingChildId) {
383+
await this.upsertCore(
384+
{ ...item, status: "active", awaitingChildId: undefined, delegatedToId: undefined },
385+
{ skipTransitionCheck: true },
386+
)
387+
console.warn(
388+
`[TaskHistoryStore] Reconciled invalid delegation: task ${item.id} → active (no awaitingChildId)`,
389+
)
390+
repairsInThisPass++
391+
continue
422392
}
423-
} while (repairsInThisPass > 0)
424-
})
393+
394+
const child = byId.get(item.awaitingChildId)
395+
396+
if (!child) {
397+
await this.upsertCore(
398+
{
399+
...item,
400+
status: "active",
401+
awaitingChildId: undefined,
402+
delegatedToId: undefined,
403+
},
404+
{ skipTransitionCheck: true },
405+
)
406+
console.warn(
407+
`[TaskHistoryStore] Reconciled orphaned delegation: task ${item.id} → active (child ${item.awaitingChildId} not found)`,
408+
)
409+
repairsInThisPass++
410+
} else if (child.status === "completed") {
411+
await this.upsertCore(
412+
{
413+
...item,
414+
status: "active",
415+
awaitingChildId: undefined,
416+
delegatedToId: undefined,
417+
completedByChildId: child.id,
418+
completionResultSummary:
419+
child.completionResultSummary ?? "Task completed (recovered after interruption)",
420+
},
421+
{ skipTransitionCheck: true },
422+
)
423+
console.warn(
424+
`[TaskHistoryStore] Reconciled interrupted handoff: task ${item.id} → active (child ${item.awaitingChildId} already completed)`,
425+
)
426+
repairsInThisPass++
427+
}
428+
// child.status === "active", "interrupted", or "delegated" → leave as-is this pass
429+
}
430+
} while (repairsInThisPass > 0)
425431
}
426432

427433
// ────────────────────────────── Cache invalidation ──────────────────────────────
@@ -458,6 +464,22 @@ export class TaskHistoryStore {
458464
* file if one doesn't already exist. This is idempotent and safe to re-run.
459465
*/
460466
async migrateFromGlobalState(taskHistoryEntries: HistoryItem[]): Promise<void> {
467+
return this.withLock(() => this.migrateFromGlobalStateCore(taskHistoryEntries))
468+
}
469+
470+
async mutateLocked<T>(fn: () => Promise<T>): Promise<T> {
471+
return this.withLock(fn)
472+
}
473+
474+
async reconcileLocked(): Promise<void> {
475+
return this.reconcileCore()
476+
}
477+
478+
async deleteManyLocked(taskIds: string[]): Promise<void> {
479+
return this.deleteManyCore(taskIds)
480+
}
481+
482+
private async migrateFromGlobalStateCore(taskHistoryEntries: HistoryItem[]): Promise<void> {
461483
if (!taskHistoryEntries || taskHistoryEntries.length === 0) {
462484
return
463485
}
@@ -495,7 +517,7 @@ export class TaskHistoryStore {
495517

496518
// Repair any delegation inconsistencies introduced by the migrated entries.
497519
// reconcileDelegationState() is idempotent so running it again is safe.
498-
await this.reconcileDelegationState()
520+
await this.reconcileDelegationStateCore()
499521
}
500522

501523
// ────────────────────────────── Private: Index management ──────────────────────────────

0 commit comments

Comments
 (0)