feat(eval): frozen-task ablation benchmark — and three guard defects it uncovered

README.md

@@ -1297,6 +1297,19 @@ pnpm clean
 - [RedBox](https://github.com/Jamailar/RedBox) - Local AI creation workspace for Xiaohongshu creators.
 - [1flowbase](https://github.com/taichuy/1flowbase) - Virtual model gateway for publishing multi-model workflows as OpenAI/Claude-compatible endpoints, with trace, token, latency, and cost visibility.
 
+## Ablation Benchmark (reproducible)
+
+A frozen 30-task benchmark measures whether SDD constraints and the hallucination guard actually improve first-pass success. It runs on an isolated fixture project with machine-checkable acceptance (`tsc --noEmit`, `vitest`, file assertions), two arms, and resume-on-interrupt.
+
+```bash
+pnpm build
+node benchmarks/eval/run-eval.mjs --arm full --tasks all       # SDD on
+node benchmarks/eval/run-eval.mjs --arm ablation --tasks all   # SDD off
+node benchmarks/eval/report.mjs benchmarks/eval/out
+```
+
+Latest results: [`benchmarks/results/`](benchmarks/results/). **The current findings are negative and actionable**: SDD showed no measurable effect on first-pass rate, and the hallucination guard recorded zero interceptions while syntactically invalid files still landed on disk. Three root causes are documented in the report and tracked as issues.
+
 ## Contributing
 
 Welcome to contribute! Submit issues, bugs, or suggestions:

benchmarks/eval/checks.mjs

@@ -0,0 +1,40 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { spawnSync } from 'node:child_process';
+
+/** 逐条执行任务 checks；返回 [{kind, ok, detail}]，全 ok 即任务 pass */
+export function runChecks(checks, { workspace, resultText }) {
+  return checks.map((check) => {
+    try {
+      switch (check.kind) {
+        case 'file_exists': {
+          const ok = existsSync(join(workspace, check.path));
+          return { kind: check.kind, ok, detail: check.path };
+        }
+        case 'file_contains': {
+          const p = join(workspace, check.path);
+          if (!existsSync(p)) return { kind: check.kind, ok: false, detail: `missing ${check.path}` };
+          const ok = new RegExp(check.pattern).test(readFileSync(p, 'utf8'));
+          return { kind: check.kind, ok, detail: check.pattern };
+        }
+        case 'result_contains': {
+          const ok = new RegExp(check.pattern, 'i').test(resultText ?? '');
+          return { kind: check.kind, ok, detail: check.pattern };
+        }
+        case 'typecheck': {
+          const r = spawnSync('npx', ['tsc', '--noEmit'], { cwd: workspace, timeout: 120000 });
+          return { kind: check.kind, ok: r.status === 0, detail: r.status === 0 ? '' : String(r.stdout).slice(0, 300) };
+        }
+        case 'cmd': {
+          const [bin, ...rest] = check.argv;
+          const r = spawnSync(bin, rest, { cwd: workspace, timeout: 180000 });
+          return { kind: check.kind, ok: r.status === 0, detail: check.argv.join(' ') };
+        }
+        default:
+          return { kind: check.kind, ok: false, detail: 'unknown check kind' };
+      }
+    } catch (error) {
+      return { kind: check.kind, ok: false, detail: String(error).slice(0, 200) };
+    }
+  });
+}

benchmarks/eval/claude-cli-backend.mjs

@@ -0,0 +1,126 @@
+import { spawn } from 'node:child_process';
+import { mkdirSync } from 'node:fs';
+
+// 空目录作为 CLI 工作目录：阻断 Claude Code 对宿主仓库 CLAUDE.md/项目上下文的注入
+const CLI_CWD = '/tmp/frontagent-eval/claude-cli-cwd';
+mkdirSync(CLI_CWD, { recursive: true });
+
+const MODEL = process.env.EVAL_MODEL ?? 'claude-haiku-4-5';
+const CALL_TIMEOUT_MS = Number(process.env.EVAL_LLM_TIMEOUT_MS ?? 180000);
+
+const tally = { calls: 0, failures: 0, inputTokens: 0, outputTokens: 0 };
+export function getUsageTally() {
+  return { ...tally };
+}
+
+function recordFailure(error) {
+  tally.failures += 1;
+  console.error(`[claude-cli-backend] call failed (#${tally.failures}): ${String(error?.message ?? error).slice(0, 200)}`);
+}
+
+function renderPrompt(messages, system) {
+  const parts = [];
+  if (system) parts.push(`[System]\n${system}`);
+  for (const m of messages ?? []) {
+    parts.push(`[${m.role}]\n${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`);
+  }
+  return parts.join('\n\n');
+}
+
+function callClaude(prompt) {
+  return new Promise((resolvePromise, reject) => {
+    // 纯文本生成后端：禁用全部工具，否则「列目录」类 prompt 会诱发 tool_use
+    // 并在 --max-turns 1 下报 error_max_turns
+    const args = [
+      '-p',
+      '--model',
+      MODEL,
+      '--output-format',
+      'json',
+      '--max-turns',
+      '1',
+      '--disallowedTools',
+      '*',
+      '--system-prompt',
+      '你是一次性文本生成后端：只依据提示词中给出的信息直接作答，没有任何工具可用，不要尝试读取文件或执行命令。',
+    ];
+    const child = spawn('claude', args, { stdio: ['pipe', 'pipe', 'pipe'], cwd: CLI_CWD });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      const err = new Error('claude CLI timeout');
+      recordFailure(err);
+      reject(err);
+    }, CALL_TIMEOUT_MS);
+    child.stdout.on('data', (d) => {
+      stdout += d;
+    });
+    child.stderr.on('data', (d) => {
+      stderr += d;
+    });
+    child.on('error', (e) => {
+      clearTimeout(timer);
+      recordFailure(e);
+      reject(e);
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code !== 0) {
+        const err = new Error(
+          `claude exit ${code}: stderr=${stderr.slice(0, 300)} stdoutHead=${stdout.slice(0, 200)} promptChars=${prompt.length}`,
+        );
+        recordFailure(err);
+        return reject(err);
+      }
+      try {
+        const parsed = JSON.parse(stdout);
+        tally.calls += 1;
+        tally.inputTokens += parsed.usage?.input_tokens ?? 0;
+        tally.outputTokens += parsed.usage?.output_tokens ?? 0;
+        if (parsed.is_error) {
+          const err = new Error(`claude error result: ${String(parsed.result).slice(0, 300)}`);
+          recordFailure(err);
+          return reject(err);
+        }
+        resolvePromise(String(parsed.result ?? ''));
+      } catch (e) {
+        const err = new Error(`bad CLI JSON: ${String(e)}; head=${stdout.slice(0, 200)}`);
+        recordFailure(err);
+        reject(err);
+      }
+    });
+    child.stdin.end(prompt);
+  });
+}
+
+function extractJson(text) {
+  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+  const body = fenced ? fenced[1] : text;
+  const start = body.indexOf('{');
+  const end = body.lastIndexOf('}');
+  if (start === -1 || end <= start) throw new Error('no JSON object in output');
+  return body.slice(start, end + 1);
+}
+
+export function createClaudeCliBackend() {
+  return {
+    name: `claude-cli:${MODEL}`,
+    async generateText({ messages, system }) {
+      return callClaude(renderPrompt(messages, system));
+    },
+    async generateObject({ messages, system, schema, maxRetries = 2 }) {
+      let lastError = '';
+      for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
+        const suffix = `\n\n只输出一个 JSON 对象：不要 markdown 代码围栏、不要解释文字。${lastError ? `上次输出未通过 schema 校验（${lastError}），请修正。` : ''}`;
+        const text = await callClaude(renderPrompt(messages, system) + suffix);
+        try {
+          return schema.parse(JSON.parse(extractJson(text)));
+        } catch (error) {
+          lastError = String(error).slice(0, 400);
+        }
+      }
+      throw new Error(`generateObject retries exhausted: ${lastError}`);
+    },
+  };
+}

benchmarks/eval/fixture/.gitignore

@@ -0,0 +1 @@
+node_modules/

benchmarks/eval/fixture/README.md

@@ -0,0 +1,10 @@
+# Eval Fixture
+
+架构消融评测的冻结夹具：一个最小 React + TypeScript 工程。
+
+- `src/components/` —— UI 组件（PascalCase 命名导出）
+- `src/hooks/` —— 自定义 hooks（`use*` 前缀）
+- `src/utils/` —— 纯函数工具与对应测试
+- `npm run typecheck` / `npm run test` 为验收门禁
+
+组件与函数的具体接口以源码为准，本 README 不重复文档。

benchmarks/eval/fixture/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "eval-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": { "typecheck": "tsc --noEmit", "test": "vitest run" },
+  "devDependencies": {
+    "@types/react": "^18.3.0",
+    "react": "^18.3.0",
+    "react-dom": "^18.3.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.0.0"
+  }
+}