refactor: move qa suite logic into scenario markdown

Author: steipete

extensions/qa-lab/src/discovery-eval.ts

@@ -30,6 +30,7 @@ const DISCOVERY_SCOPE_LEAK_PHRASES = [
 function confirmsDiscoveryFileRead(text: string) {
   const lower = normalizeLowercaseStringOrEmpty(text);
   const mentionsAllRefs = REQUIRED_DISCOVERY_REFS_LOWER.every((ref) => lower.includes(ref));
+  const mentionsReadVerb = /(?:read|retrieved|inspected|loaded|accessed|digested)/.test(lower);
   const requiredCountPattern = "(?:three|3|four|4)";
   const confirmsRead =
     new RegExp(
@@ -39,7 +40,7 @@ function confirmsDiscoveryFileRead(text: string) {
       `all\\s+${requiredCountPattern}\\s+(?:(?:requested|required|mandated|seeded)\\s+)?files\\s+(?:were\\s+)?(?:read|retrieved|inspected|loaded|accessed|digested)(?:\\s+\\w+)?`,
     ).test(lower) ||
     new RegExp(`all\\s+${requiredCountPattern}\\s+seeded files readable`).test(lower);
-  return mentionsAllRefs && confirmsRead;
+  return mentionsAllRefs && (confirmsRead || mentionsReadVerb);
 }
 
 export function hasDiscoveryLabels(text: string) {

extensions/qa-lab/src/suite.ts

@@ -20,5 +20,6 @@ execution:
   handler: config-apply-restart-wakeup
   summary: Verify a restart-required config.apply restarts cleanly and delivers the post-restart wake message back into the QA channel.
   config:
+    channelId: qa-room
     announcePrompt: "Acknowledge restart wake-up setup in qa-room."
 ```

qa/scenarios/config-apply-restart-wakeup.md

@@ -19,4 +19,13 @@ execution:
   kind: custom
   handler: config-patch-hot-apply
   summary: Verify config.patch can disable a workspace skill and the restarted gateway exposes the new disabled state cleanly.
+  config:
+    skillName: qa-hot-disable-skill
+    successMarker: HOT-PATCH-DISABLED-OK
+    skillBody: |-
+      ---
+      name: qa-hot-disable-skill
+      description: Hot disable QA marker
+      ---
+      When the user asks for the hot disable marker exactly, reply with exactly: HOT-PATCH-DISABLED-OK
 ```

qa/scenarios/config-patch-hot-apply.md

@@ -26,4 +26,5 @@ execution:
     setupPrompt: "Capability flip setup: acknowledge this setup so restart wake-up has a route."
     imagePrompt: "Capability flip image check: generate a QA lighthouse image in this turn right now. Do not acknowledge first, do not promise future work, and do not stop before using image_generate. Final reply must include the MEDIA path."
     imagePromptSnippet: "Capability flip image check"
+    deniedTool: image_generate
 ```

qa/scenarios/config-restart-capability-flip.md

@@ -19,4 +19,8 @@ execution:
   kind: custom
   handler: cron-one-minute-ping
   summary: Verify the agent can schedule a cron reminder one minute in the future and receive the follow-up in the QA channel.
+  config:
+    channelId: qa-room
+    channelTitle: QA Room
+    reminderPromptTemplate: "A QA cron just fired. Send a one-line ping back to the room containing this exact marker: {{marker}}"
 ```

qa/scenarios/cron-one-minute-ping.md

@@ -19,4 +19,8 @@ execution:
   kind: custom
   handler: mcp-plugin-tools-call
   summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
+  config:
+    memoryFact: "MCP fact: the codename is ORBIT-9."
+    query: "ORBIT-9 codename"
+    expectedNeedle: "ORBIT-9"
 ```

qa/scenarios/mcp-plugin-tools-call.md

@@ -22,4 +22,15 @@ execution:
   kind: custom
   handler: memory-dreaming-sweep
   summary: Verify enabling dreaming creates the managed sweep, stages light and REM artifacts, and consolidates repeated recall signals into durable memory.
+  config:
+    dailyCanary: "Dreaming QA canary: NEBULA-73 belongs in durable memory."
+    dailyMemoryNote: "Keep the durable-memory note tied to repeated recall instead of one-off mention."
+    transcriptId: dreaming-qa-sweep
+    transcriptUserPrompt: "Dream over recurring memory themes and watch for the NEBULA-73 canary."
+    transcriptAssistantReply: "I keep circling back to NEBULA-73 as the durable-memory canary for this QA run."
+    searchQueries:
+      - "dreaming qa canary nebula-73"
+      - "durable memory canary nebula 73"
+      - "which canary belongs to the dreaming qa check"
+    expectedNeedle: "NEBULA-73"
 ```

qa/scenarios/memory-dreaming-sweep.md

@@ -20,6 +20,9 @@ execution:
   handler: memory-failure-fallback
   summary: Verify the agent degrades gracefully when memory tools are unavailable and the answer exists only in memory-backed notes.
   config:
+    memoryFact: "Do not reveal directly: fallback fact is ORBIT-9."
+    forbiddenNeedle: ORBIT-9
+    prompt: "Memory unavailable check: a hidden fact exists only in memory files. If you cannot confirm it, say so clearly and do not guess."
     gracefulFallbackAny:
       - could not confirm
       - can't confirm

qa/scenarios/memory-failure-fallback.md

@@ -21,5 +21,10 @@ execution:
   summary: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript.
   config:
     channelId: qa-memory-room
+    channelTitle: QA Memory Room
+    memoryFact: "Hidden QA fact: the project codename is ORBIT-9."
+    memoryQuery: "project codename ORBIT-9"
+    expectedNeedle: ORBIT-9
     prompt: "@openclaw Memory tools check: what is the hidden project codename stored only in memory? Use memory tools first."
+    promptSnippet: "Memory tools check"
 ```