test: add regression coverage and align docs/examples with quality gates

Add workflow/problem regression tests for gate/order/signal behaviors and update README/examples to document and demonstrate the stricter quality-gate contract.

Made-with: Cursor

Author: SummerOneTwo

README.md

@@ -125,7 +125,7 @@ validator_build / checker_build
 - 交互题必须先完成可用的 `interactor_build`。
 - `stress_test_run` 必须完整跑完所有轮次。
 - `problem_generate_tests` 前必须通过 `problem_validate`。
-- `problem_pack_polygon` 前必须通过 `problem_verify_tests`。
+- `problem_pack_polygon` 前必须通过 `problem_verify_tests`，并满足门禁要求的结构化质量信号（如 `limit_semantics`、`wrong_solution_kill`、`validator_check`）。
 - 生成最终测试后会自动清除旧的 `tests_verified` 状态，必须重新验证。
 
 ## 题目目录和 manifest

examples/checker-sample/autocode.json

@@ -6,6 +6,15 @@
   "memory_limit_mb": 256,
   "statement_path": "statements/README.md",
   "tutorial_path": "statements/tutorial.md",
+  "quality_gates": {
+    "require_stress_passed": true,
+    "require_validation_passed": true,
+    "require_tests_verified": true,
+    "require_limit_semantics": true,
+    "require_wrong_solution_kill": true,
+    "require_validator_check": true,
+    "min_limit_case_ratio": 0.5
+  },
   "solutions": [
     {"name": "sol", "role": "main", "language": "cpp", "path": "solutions/sol.cpp"},
     {"name": "brute", "role": "brute", "language": "cpp", "path": "solutions/brute.cpp"},

examples/exact-sample/autocode.json

@@ -9,6 +9,15 @@
   "constraints": {
     "n_max": 100000
   },
+  "quality_gates": {
+    "require_stress_passed": true,
+    "require_validation_passed": true,
+    "require_tests_verified": true,
+    "require_limit_semantics": true,
+    "require_wrong_solution_kill": true,
+    "require_validator_check": true,
+    "min_limit_case_ratio": 0.5
+  },
   "solutions": [
     {"name": "sol", "role": "main", "language": "cpp", "path": "solutions/sol.cpp"},
     {"name": "brute", "role": "brute", "language": "cpp", "path": "solutions/brute.cpp"}

examples/interactive-sample/autocode.json

@@ -6,6 +6,15 @@
   "memory_limit_mb": 256,
   "statement_path": "statements/README.md",
   "tutorial_path": "statements/tutorial.md",
+  "quality_gates": {
+    "require_stress_passed": true,
+    "require_validation_passed": true,
+    "require_tests_verified": true,
+    "require_limit_semantics": true,
+    "require_wrong_solution_kill": true,
+    "require_validator_check": false,
+    "min_limit_case_ratio": 0.5
+  },
   "solutions": [
     {"name": "sol", "role": "main", "language": "cpp", "path": "solutions/sol.cpp"},
     {"name": "brute", "role": "brute", "language": "cpp", "path": "solutions/brute.cpp"}

tests/test_tools/test_problem.py

@@ -23,6 +23,23 @@
 from autocode_mcp.utils.platform import get_exe_extension
 
 
+def write_verified_workflow_state(problem_dir: str) -> None:
+    workflow_dir = os.path.join(problem_dir, ".autocode-workflow")
+    os.makedirs(workflow_dir, exist_ok=True)
+    with open(os.path.join(workflow_dir, "state.json"), "w", encoding="utf-8") as f:
+        json.dump(
+            {
+                "tests_verified": True,
+                "verify_signals": {
+                    "limit_semantics": {"executed": True, "passed": True},
+                    "wrong_solution_kill": {"executed": True, "passed": True},
+                    "validator_check": {"executed": True, "passed": True},
+                },
+            },
+            f,
+        )
+
+
 @pytest.mark.asyncio
 async def test_problem_create():
     """测试题目目录创建。"""
@@ -87,6 +104,7 @@ async def test_problem_pack_polygon():
             f.write("1\n")
         with open(os.path.join(problem_dir, "tests", "01.ans"), "w", encoding="utf-8") as f:
             f.write("1\n")
+        write_verified_workflow_state(problem_dir)
 
         # 打包
         result = await pack_tool.execute(problem_dir=problem_dir)
@@ -113,6 +131,7 @@ async def test_problem_pack_polygon_creates_xml():
             f.write("1\n")
         with open(os.path.join(problem_dir, "tests", "01.ans"), "w", encoding="utf-8") as f:
             f.write("1\n")
+        write_verified_workflow_state(problem_dir)
 
         result = await tool.execute(
             problem_dir=problem_dir,
@@ -926,6 +945,7 @@ async def test_problem_pack_polygon_dynamic_test_count():
                 f.write(f"answer {i}\n")
         with open(os.path.join(problem_dir, "sol.cpp"), "w", encoding="utf-8") as f:
             f.write("// sol\n")
+        write_verified_workflow_state(problem_dir)
 
         await pack_tool.execute(problem_dir=problem_dir)
 
@@ -955,6 +975,7 @@ async def test_problem_pack_polygon_sanitizes_answer_ext_from_manifest():
             json.dump({"answer_ext": ".bad<ext>"}, f)
         with open(os.path.join(problem_dir, "sol.cpp"), "w", encoding="utf-8") as f:
             f.write("// sol\n")
+        write_verified_workflow_state(problem_dir)
 
         result = await pack_tool.execute(problem_dir=problem_dir)
         assert result.success
@@ -1011,6 +1032,27 @@ async def test_problem_pack_polygon_fails_when_workflow_state_unverified():
         assert "run problem_verify_tests first" in result.error
 
 
+@pytest.mark.asyncio
+async def test_problem_pack_polygon_fails_when_workflow_state_missing():
+    tool = ProblemPackPolygonTool()
+    with tempfile.TemporaryDirectory() as tmpdir:
+        problem_dir = os.path.join(tmpdir, "pack_state_missing")
+        os.makedirs(os.path.join(problem_dir, "tests"), exist_ok=True)
+        os.makedirs(os.path.join(problem_dir, "statements"), exist_ok=True)
+        os.makedirs(os.path.join(problem_dir, "solutions"), exist_ok=True)
+        with open(os.path.join(problem_dir, "tests", "01.in"), "w", encoding="utf-8") as f:
+            f.write("1\n")
+        with open(os.path.join(problem_dir, "tests", "01.ans"), "w", encoding="utf-8") as f:
+            f.write("1\n")
+        with open(os.path.join(problem_dir, "statements", "README.md"), "w", encoding="utf-8") as f:
+            f.write("# T\n")
+        with open(os.path.join(problem_dir, "solutions", "sol.cpp"), "w", encoding="utf-8") as f:
+            f.write("// sol\n")
+        result = await tool.execute(problem_dir=problem_dir)
+        assert not result.success
+        assert "workflow state missing" in result.error
+
+
 @pytest.mark.asyncio
 async def test_problem_pack_polygon_respects_require_tests_verified_override():
     tool = ProblemPackPolygonTool()
@@ -1029,7 +1071,18 @@ async def test_problem_pack_polygon_respects_require_tests_verified_override():
         with open(os.path.join(problem_dir, "solutions", "sol.cpp"), "w", encoding="utf-8") as f:
             f.write("// sol\n")
         with open(os.path.join(problem_dir, "autocode.json"), "w", encoding="utf-8") as f:
-            json.dump({"problem_name": "t", "quality_gates": {"require_tests_verified": False}}, f)
+            json.dump(
+                {
+                    "problem_name": "t",
+                    "quality_gates": {
+                        "require_tests_verified": False,
+                        "require_limit_semantics": False,
+                        "require_wrong_solution_kill": False,
+                        "require_validator_check": False,
+                    },
+                },
+                f,
+            )
         with open(
             os.path.join(problem_dir, ".autocode-workflow", "state.json"),
             "w",
@@ -1040,6 +1093,30 @@ async def test_problem_pack_polygon_respects_require_tests_verified_override():
         assert result.success
 
 
+@pytest.mark.asyncio
+async def test_problem_pack_polygon_fails_when_required_verify_signal_missing():
+    tool = ProblemPackPolygonTool()
+    with tempfile.TemporaryDirectory() as tmpdir:
+        problem_dir = os.path.join(tmpdir, "pack_signal_missing")
+        os.makedirs(os.path.join(problem_dir, "tests"), exist_ok=True)
+        os.makedirs(os.path.join(problem_dir, "statements"), exist_ok=True)
+        os.makedirs(os.path.join(problem_dir, "solutions"), exist_ok=True)
+        os.makedirs(os.path.join(problem_dir, ".autocode-workflow"), exist_ok=True)
+        with open(os.path.join(problem_dir, "tests", "01.in"), "w", encoding="utf-8") as f:
+            f.write("1\n")
+        with open(os.path.join(problem_dir, "tests", "01.ans"), "w", encoding="utf-8") as f:
+            f.write("1\n")
+        with open(os.path.join(problem_dir, "statements", "README.md"), "w", encoding="utf-8") as f:
+            f.write("# T\n")
+        with open(os.path.join(problem_dir, "solutions", "sol.cpp"), "w", encoding="utf-8") as f:
+            f.write("// sol\n")
+        with open(os.path.join(problem_dir, ".autocode-workflow", "state.json"), "w", encoding="utf-8") as f:
+            json.dump({"tests_verified": True, "verify_signals": {}}, f)
+        result = await tool.execute(problem_dir=problem_dir)
+        assert not result.success
+        assert "verification signal" in result.error
+
+
 @pytest.mark.asyncio
 async def test_problem_pack_polygon_enforces_min_limit_case_ratio():
     tool = ProblemPackPolygonTool()
@@ -1070,6 +1147,7 @@ async def test_problem_pack_polygon_enforces_min_limit_case_ratio():
             f.write("# T\n")
         with open(os.path.join(problem_dir, "solutions", "sol.cpp"), "w", encoding="utf-8") as f:
             f.write("// sol\n")
+        write_verified_workflow_state(problem_dir)
         with open(
             os.path.join(problem_dir, "autocode.json"),
             "w",

tests/test_workflow_guard.py

@@ -171,7 +171,10 @@ def test_pre_tool_denies_interactive_generator_before_interactor(tmp_path, capsy
     assert exit_code == 0
     parsed = json.loads(captured)
     assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny"
-    assert "interactor_build" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
+    assert (
+        "interactor_build" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
+        or "solution_analyze" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
+    )
 
 
 def test_pre_tool_denies_pack_before_tests_verified(tmp_path, capsys):
@@ -377,6 +380,59 @@ def test_pre_tool_enforces_solution_analyze_before_validator_build(tmp_path, cap
     assert "solution_analyze" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
 
 
+def test_pre_tool_enforces_solution_audits_before_validator_build(tmp_path, capsys):
+    module = load_module()
+    problem_dir = tmp_path / "problem"
+    (problem_dir / "files").mkdir(parents=True)
+    (problem_dir / "solutions").mkdir(parents=True)
+    write_manifest(problem_dir)
+    state = module.infer_state(str(problem_dir))
+    state["created"] = True
+    state["sol_built"] = True
+    state["brute_built"] = True
+    state["solution_analyzed"] = True
+    state["std_audited"] = False
+    state["brute_audited"] = False
+    module.save_state(str(problem_dir), state)
+
+    payload = {
+        "tool_name": "mcp__autocode__validator_build",
+        "tool_input": {"problem_dir": str(problem_dir)},
+    }
+    module.pre_tool(payload)
+    captured = capsys.readouterr().out
+    parsed = json.loads(captured)
+    assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny"
+    assert "solution_audit_std" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
+
+
+def test_pre_tool_enforces_solution_audits_before_generator_build(tmp_path, capsys):
+    module = load_module()
+    problem_dir = tmp_path / "problem"
+    (problem_dir / "files").mkdir(parents=True)
+    (problem_dir / "solutions").mkdir(parents=True)
+    write_manifest(problem_dir)
+    state = module.infer_state(str(problem_dir))
+    state["created"] = True
+    state["sol_built"] = True
+    state["brute_built"] = True
+    state["solution_analyzed"] = True
+    state["validator_ready"] = True
+    state["validator_accuracy"] = 1.0
+    state["std_audited"] = True
+    state["brute_audited"] = False
+    module.save_state(str(problem_dir), state)
+    payload = {
+        "tool_name": "mcp__autocode__generator_build",
+        "tool_input": {"problem_dir": str(problem_dir)},
+    }
+    module.pre_tool(payload)
+    captured = capsys.readouterr().out
+    parsed = json.loads(captured)
+    assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny"
+    assert "solution_audit_std" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
+
+
 def test_pre_tool_pack_respects_quality_gate_override(tmp_path, capsys):
     module = load_module()
     problem_dir = tmp_path / "problem"
@@ -388,13 +444,17 @@ def test_pre_tool_pack_respects_quality_gate_override(tmp_path, capsys):
             "require_stress_passed": True,
             "require_validation_passed": True,
             "require_tests_verified": False,
+            "require_limit_semantics": False,
+            "require_wrong_solution_kill": False,
+            "require_validator_check": False,
             "min_limit_case_ratio": 0.5,
         },
     )
     state = module.infer_state(str(problem_dir))
     state["tests_generated"] = True
     state["generated_test_count"] = 1
     state["tests_verified"] = False
+    state["verify_signals"] = {}
     module.save_state(str(problem_dir), state)
 
     payload = {
@@ -444,6 +504,59 @@ def test_post_tool_verify_tests_applies_min_limit_case_ratio(tmp_path):
     assert state["limit_case_ratio"] == 0.5
 
 
+def test_post_tool_verify_tests_persists_quality_signals_and_history(tmp_path):
+    module = load_module()
+    problem_dir = tmp_path / "problem"
+    (problem_dir / "files").mkdir(parents=True)
+    (problem_dir / "solutions").mkdir(parents=True)
+    write_manifest(problem_dir)
+    payload = {
+        "tool_name": "mcp__autocode__problem_verify_tests",
+        "tool_input": {"problem_dir": str(problem_dir)},
+        "tool_response": {
+            "structuredContent": {
+                "success": True,
+                "data": {
+                    "passed": True,
+                    "quality_signals": {
+                        "limit_semantics": {"executed": True, "passed": True},
+                        "wrong_solution_kill": {"executed": True, "passed": True},
+                        "validator_check": {"executed": True, "passed": True},
+                    },
+                },
+            }
+        },
+    }
+    module.post_tool(payload)
+    state = module.load_state(str(problem_dir))
+    assert state["verify_signals"]["limit_semantics"]["passed"] is True
+    assert isinstance(state.get("history"), list)
+    assert state["history"][-1]["tool"] == "problem_verify_tests"
+
+
+def test_pre_tool_pack_denies_when_required_verify_signal_missing(tmp_path, capsys):
+    module = load_module()
+    problem_dir = tmp_path / "problem"
+    (problem_dir / "files").mkdir(parents=True)
+    (problem_dir / "solutions").mkdir(parents=True)
+    write_manifest(problem_dir)
+    state = module.infer_state(str(problem_dir))
+    state["tests_generated"] = True
+    state["generated_test_count"] = 2
+    state["tests_verified"] = True
+    state["verify_signals"] = {"limit_semantics": {"executed": True, "passed": False}}
+    module.save_state(str(problem_dir), state)
+    payload = {
+        "tool_name": "mcp__autocode__problem_pack_polygon",
+        "tool_input": {"problem_dir": str(problem_dir)},
+    }
+    module.pre_tool(payload)
+    captured = capsys.readouterr().out
+    parsed = json.loads(captured)
+    assert parsed["hookSpecificOutput"]["permissionDecision"] == "deny"
+    assert "limit_semantics" in parsed["hookSpecificOutput"]["permissionDecisionReason"]
+
+
 def test_pre_tool_pack_denies_when_limit_ratio_below_gate(tmp_path, capsys):
     module = load_module()
     problem_dir = tmp_path / "problem"