Skip to content

Commit 9006817

Browse files
vincentkocvincentkoc
authored
test(qa-lab): seed broken-turn recovery scenarios (openclaw#66416)
1 parent 37f449d commit 9006817

7 files changed

Lines changed: 365 additions & 2 deletions

‎extensions/qa-lab/src/scenario-catalog.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,30 @@ describe("qa scenario catalog", () => {
118118
);
119119
});
120120

121+
it("includes the seeded mock-only broken-turn scenarios in the markdown pack", () => {
122+
const scenarioIds = [
123+
"reasoning-only-recovery-replay-safe-read",
124+
"reasoning-only-no-auto-retry-after-write",
125+
"empty-response-recovery-replay-safe-read",
126+
"empty-response-retry-budget-exhausted",
127+
];
128+
129+
for (const scenarioId of scenarioIds) {
130+
const scenario = readQaScenarioById(scenarioId);
131+
const config = readQaScenarioExecutionConfig(scenarioId) as
132+
| {
133+
requiredProvider?: string;
134+
prompt?: string;
135+
}
136+
| undefined;
137+
138+
expect(scenario.sourcePath).toBe(`qa/scenarios/${scenarioId}.md`);
139+
expect(config?.requiredProvider).toBe("mock-openai");
140+
expect(config?.prompt).toContain("check");
141+
expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0);
142+
}
143+
});
144+
121145
it("keeps mock-only image debug assertions guarded in live-frontier runs", () => {
122146
const scenario = readQaScenarioPack().scenarios.find(
123147
(candidate) => candidate.id === "image-understanding-attachment",

‎qa/scenarios/empty-response-recovery-replay-safe-read.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Empty-response recovery after replay-safe read
2+
3+
```yaml qa-scenario
4+
id: empty-response-recovery-replay-safe-read
5+
title: Empty-response recovery after replay-safe read
6+
surface: runtime
7+
objective: Verify an empty visible GPT turn after a replay-safe read auto-continues into a visible answer.
8+
successCriteria:
9+
- Scenario is mock-openai only so live lanes do not pick it up implicitly.
10+
- The agent performs a replay-safe read before the empty response.
11+
- The runtime injects the visible-answer continuation instruction after the empty turn.
12+
- The final visible reply contains the exact recovery marker.
13+
docsRefs:
14+
- docs/help/testing.md
15+
codeRefs:
16+
- extensions/qa-lab/src/mock-openai-server.ts
17+
- src/agents/pi-embedded-runner/run/incomplete-turn.ts
18+
execution:
19+
kind: flow
20+
summary: Verify empty OpenAI turns recover after a replay-safe read.
21+
config:
22+
requiredProvider: mock-openai
23+
promptSnippet: Empty response continuation QA check
24+
prompt: "Empty response continuation QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-RECOVERED-OK."
25+
expectedReply: EMPTY-RECOVERED-OK
26+
retryNeedle: The previous attempt did not produce a user-visible answer.
27+
```
28+
29+
```yaml qa-flow
30+
steps:
31+
- name: retries an empty replay-safe read into a visible answer
32+
actions:
33+
- assert:
34+
expr: "env.providerMode === 'mock-openai'"
35+
message: this seeded scenario is mock-openai only
36+
- call: waitForGatewayHealthy
37+
args:
38+
- ref: env
39+
- 60000
40+
- call: reset
41+
- set: requestCountBefore
42+
value:
43+
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
44+
- set: sessionKey
45+
value:
46+
expr: "`agent:qa:empty-response-recovery:${randomUUID().slice(0, 8)}`"
47+
- call: runAgentPrompt
48+
args:
49+
- ref: env
50+
- sessionKey:
51+
ref: sessionKey
52+
message:
53+
expr: config.prompt
54+
timeoutMs:
55+
expr: liveTurnTimeoutMs(env, 45000)
56+
- call: waitForOutboundMessage
57+
saveAs: outbound
58+
args:
59+
- ref: state
60+
- lambda:
61+
params: [candidate]
62+
expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)"
63+
- expr: liveTurnTimeoutMs(env, 30000)
64+
- assert:
65+
expr: "outbound.text.includes(config.expectedReply)"
66+
message:
67+
expr: "`missing empty-response recovery marker: ${outbound.text}`"
68+
- if:
69+
expr: "Boolean(env.mock)"
70+
then:
71+
- set: scenarioRequests
72+
value:
73+
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
74+
- assert:
75+
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')"
76+
message: expected replay-safe read request in mock trace
77+
- assert:
78+
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.retryNeedle))"
79+
message: expected empty-response retry instruction in mock trace
80+
detailsExpr: "env.mock ? `${outbound.text}\\nrequests=${String(scenarioRequests?.length ?? 0)}` : outbound.text"
81+
```

‎qa/scenarios/empty-response-retry-budget-exhausted.md‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# Empty-response retry budget exhausted
2+
3+
```yaml qa-scenario
4+
id: empty-response-retry-budget-exhausted
5+
title: Empty-response retry budget exhausted
6+
surface: runtime
7+
objective: Verify repeated empty GPT turns exhaust the retry budget after one continuation attempt.
8+
successCriteria:
9+
- Scenario is mock-openai only so live lanes do not pick it up implicitly.
10+
- The agent performs the replay-safe read that makes retrying allowed.
11+
- Mock trace shows the run reaches a terminal post-read turn without ever producing the requested success marker.
12+
docsRefs:
13+
- docs/help/testing.md
14+
codeRefs:
15+
- extensions/qa-lab/src/mock-openai-server.ts
16+
- src/agents/pi-embedded-runner/run/incomplete-turn.ts
17+
execution:
18+
kind: flow
19+
summary: Verify empty-response retry exhaustion still surfaces a visible failure.
20+
config:
21+
requiredProvider: mock-openai
22+
promptSnippet: Empty response exhaustion QA check
23+
prompt: "Empty response exhaustion QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-EXHAUSTED-OK."
24+
retryNeedle: The previous attempt did not produce a user-visible answer.
25+
```
26+
27+
```yaml qa-flow
28+
steps:
29+
- name: surfaces a retry error after empty-response exhaustion
30+
actions:
31+
- assert:
32+
expr: "env.providerMode === 'mock-openai'"
33+
message: this seeded scenario is mock-openai only
34+
- call: waitForGatewayHealthy
35+
args:
36+
- ref: env
37+
- 60000
38+
- call: reset
39+
- set: requestCountBefore
40+
value:
41+
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
42+
- set: sessionKey
43+
value:
44+
expr: "`agent:qa:empty-response-exhausted:${randomUUID().slice(0, 8)}`"
45+
- call: startAgentRun
46+
saveAs: started
47+
args:
48+
- ref: env
49+
- sessionKey:
50+
ref: sessionKey
51+
message:
52+
expr: config.prompt
53+
timeoutMs:
54+
expr: liveTurnTimeoutMs(env, 45000)
55+
- set: waited
56+
value:
57+
expr: "await env.gateway.call('agent.wait', { runId: started.runId, timeoutMs: liveTurnTimeoutMs(env, 45000) }, { timeoutMs: liveTurnTimeoutMs(env, 50000) })"
58+
- assert:
59+
expr: "waited?.status === 'ok'"
60+
message:
61+
expr: "`agent.wait returned ${String(waited?.status ?? 'unknown')}: ${String(waited?.error ?? '')}`"
62+
- if:
63+
expr: "Boolean(env.mock)"
64+
then:
65+
- set: scenarioRequests
66+
value:
67+
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
68+
- assert:
69+
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')"
70+
message: expected replay-safe read request in mock trace
71+
- assert:
72+
expr: "scenarioRequests.length >= 2"
73+
message: expected at least the replay-safe read request and one terminal post-read turn
74+
detailsExpr: "env.mock ? `requests=${String(scenarioRequests?.length ?? 0)}` : String(waited?.status ?? '')"
75+
```

‎qa/scenarios/reasoning-only-no-auto-retry-after-write.md‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Reasoning-only no-auto-retry after write
2+
3+
```yaml qa-scenario
4+
id: reasoning-only-no-auto-retry-after-write
5+
title: Reasoning-only no-auto-retry after write
6+
surface: runtime
7+
objective: Verify a GPT-style reasoning-only turn after a mutating write stays replay-unsafe and does not auto-retry.
8+
successCriteria:
9+
- Scenario is mock-openai only so live lanes do not pick it up implicitly.
10+
- The agent performs the seeded mutating write.
11+
- Mock trace does not include an automatic reasoning-only retry instruction.
12+
- Mock trace stops after the write-side reasoning-only terminal turn instead of attempting a continuation.
13+
docsRefs:
14+
- docs/help/testing.md
15+
- docs/help/gpt54-codex-agentic-parity.md
16+
codeRefs:
17+
- extensions/qa-lab/src/mock-openai-server.ts
18+
- src/agents/pi-embedded-runner/run/incomplete-turn.ts
19+
execution:
20+
kind: flow
21+
summary: Verify reasoning-only turns after a write do not auto-retry.
22+
config:
23+
requiredProvider: mock-openai
24+
promptSnippet: Reasoning-only after write safety check
25+
prompt: "Reasoning-only after write safety check: write reasoning-only-side-effect.txt, then answer with exactly SIDE-EFFECT-GUARD-OK."
26+
retryNeedle: recorded reasoning but did not produce a user-visible answer
27+
outputFile: reasoning-only-side-effect.txt
28+
```
29+
30+
```yaml qa-flow
31+
steps:
32+
- name: keeps replay-unsafety explicit after a mutating write
33+
actions:
34+
- assert:
35+
expr: "env.providerMode === 'mock-openai'"
36+
message: this seeded scenario is mock-openai only
37+
- call: waitForGatewayHealthy
38+
args:
39+
- ref: env
40+
- 60000
41+
- call: reset
42+
- set: requestCountBefore
43+
value:
44+
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
45+
- set: sessionKey
46+
value:
47+
expr: "`agent:qa:reasoning-only-write:${randomUUID().slice(0, 8)}`"
48+
- call: startAgentRun
49+
saveAs: started
50+
args:
51+
- ref: env
52+
- sessionKey:
53+
ref: sessionKey
54+
message:
55+
expr: config.prompt
56+
timeoutMs:
57+
expr: liveTurnTimeoutMs(env, 45000)
58+
- set: waited
59+
value:
60+
expr: "await env.gateway.call('agent.wait', { runId: started.runId, timeoutMs: liveTurnTimeoutMs(env, 45000) }, { timeoutMs: liveTurnTimeoutMs(env, 50000) })"
61+
- assert:
62+
expr: "waited?.status === 'ok'"
63+
message:
64+
expr: "`agent.wait returned ${String(waited?.status ?? 'unknown')}: ${String(waited?.error ?? '')}`"
65+
- call: fs.readFile
66+
saveAs: sideEffect
67+
args:
68+
- expr: "path.join(env.gateway.workspaceDir, config.outputFile)"
69+
- utf8
70+
- assert:
71+
expr: "sideEffect.includes('side effects already happened')"
72+
message:
73+
expr: "`side-effect file missing expected contents: ${sideEffect}`"
74+
- if:
75+
expr: "Boolean(env.mock)"
76+
then:
77+
- set: scenarioRequests
78+
value:
79+
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
80+
- assert:
81+
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'write')"
82+
message: expected mutating write request in mock trace
83+
- assert:
84+
expr: "!scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.retryNeedle))"
85+
message: reasoning-only retry instruction should not be injected after a write
86+
- assert:
87+
expr: "scenarioRequests.filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).length === 2"
88+
message: expected exactly the write request plus the reasoning-only terminal request
89+
detailsExpr: "env.mock ? `requests=${String(scenarioRequests?.length ?? 0)} sideEffect=${sideEffect.trim()}` : sideEffect"
90+
```

‎qa/scenarios/reasoning-only-recovery-replay-safe-read.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Reasoning-only recovery after replay-safe read
2+
3+
```yaml qa-scenario
4+
id: reasoning-only-recovery-replay-safe-read
5+
title: Reasoning-only recovery after replay-safe read
6+
surface: runtime
7+
objective: Verify a GPT-style reasoning-only turn after a replay-safe read auto-continues into a visible answer.
8+
successCriteria:
9+
- Scenario is mock-openai only so live lanes do not pick it up implicitly.
10+
- The agent performs a replay-safe read before the reasoning-only turn.
11+
- The runtime injects the visible-answer continuation instruction after the reasoning-only turn.
12+
- The final visible reply contains the exact recovery marker.
13+
docsRefs:
14+
- docs/help/testing.md
15+
codeRefs:
16+
- extensions/qa-lab/src/mock-openai-server.ts
17+
- src/agents/pi-embedded-runner/run/incomplete-turn.ts
18+
execution:
19+
kind: flow
20+
summary: Verify reasoning-only OpenAI turns recover after a replay-safe read.
21+
config:
22+
requiredProvider: mock-openai
23+
promptSnippet: Reasoning-only continuation QA check
24+
prompt: "Reasoning-only continuation QA check: read QA_KICKOFF_TASK.md, then answer with exactly REASONING-RECOVERED-OK."
25+
expectedReply: REASONING-RECOVERED-OK
26+
retryNeedle: recorded reasoning but did not produce a user-visible answer
27+
```
28+
29+
```yaml qa-flow
30+
steps:
31+
- name: retries a replay-safe read into a visible answer
32+
actions:
33+
- assert:
34+
expr: "env.providerMode === 'mock-openai'"
35+
message: this seeded scenario is mock-openai only
36+
- call: waitForGatewayHealthy
37+
args:
38+
- ref: env
39+
- 60000
40+
- call: reset
41+
- set: requestCountBefore
42+
value:
43+
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
44+
- set: sessionKey
45+
value:
46+
expr: "`agent:qa:reasoning-only-recovery:${randomUUID().slice(0, 8)}`"
47+
- call: runAgentPrompt
48+
args:
49+
- ref: env
50+
- sessionKey:
51+
ref: sessionKey
52+
message:
53+
expr: config.prompt
54+
timeoutMs:
55+
expr: liveTurnTimeoutMs(env, 45000)
56+
- call: waitForOutboundMessage
57+
saveAs: outbound
58+
args:
59+
- ref: state
60+
- lambda:
61+
params: [candidate]
62+
expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.expectedReply)"
63+
- expr: liveTurnTimeoutMs(env, 30000)
64+
- assert:
65+
expr: "outbound.text.includes(config.expectedReply)"
66+
message:
67+
expr: "`missing recovery marker: ${outbound.text}`"
68+
- if:
69+
expr: "Boolean(env.mock)"
70+
then:
71+
- set: scenarioRequests
72+
value:
73+
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBefore)"
74+
- assert:
75+
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.promptSnippet) && request.plannedToolName === 'read')"
76+
message: expected replay-safe read request in mock trace
77+
- assert:
78+
expr: "scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.retryNeedle))"
79+
message: expected reasoning-only retry instruction in mock trace
80+
detailsExpr: "env.mock ? `${outbound.text}\\nrequests=${String(scenarioRequests?.length ?? 0)}` : outbound.text"
81+
```

‎src/agents/execution-contract.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,16 @@ describe("resolveEffectiveExecutionContract", () => {
2121
).toBe("strict-agentic");
2222
});
2323

24+
it("auto-activates on the mock-openai qa lane", () => {
25+
expect(
26+
resolveEffectiveExecutionContract({
27+
config: emptyConfig,
28+
provider: "mock-openai",
29+
modelId: "mock-openai/gpt-5.4",
30+
}),
31+
).toBe("strict-agentic");
32+
});
33+
2434
it("auto-activates on gpt-5o and variants without a separator", () => {
2535
for (const modelId of ["gpt-5", "gpt-5o", "gpt-5o-mini"]) {
2636
expect(

‎src/agents/execution-contract.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,16 @@ const STRICT_AGENTIC_MODEL_ID_PATTERN = /^gpt-5(?:[.o-]|$)/i;
3939
* Supported provider + model combinations where strict-agentic is the intended
4040
* runtime contract. Kept as a narrow helper so both the execution-contract
4141
* resolver and the `update_plan` auto-enable gate converge on the same
42-
* definition of "GPT-5-family openai/openai-codex run".
42+
* definition of "GPT-5-family openai/openai-codex run". The embedded
43+
* `mock-openai` QA lane intentionally piggybacks on that contract so repo QA
44+
* can exercise the same incomplete-turn recovery rules end to end.
4345
*/
4446
export function isStrictAgenticSupportedProviderModel(params: {
4547
provider?: string | null;
4648
modelId?: string | null;
4749
}): boolean {
4850
const provider = normalizeLowercaseStringOrEmpty(params.provider ?? "");
49-
if (provider !== "openai" && provider !== "openai-codex") {
51+
if (provider !== "openai" && provider !== "openai-codex" && provider !== "mock-openai") {
5052
return false;
5153
}
5254
const modelId = typeof params.modelId === "string" ? params.modelId : "";

0 commit comments

Comments
 (0)