fix(tests): add tracking comments to _KNOWN_NON_CONFORMING_RULES and enforce

- Add # tracking: #4069 comments to agent-eval and skill-eval entries
  in _KNOWN_NON_CONFORMING_RULES
- Reduce _ALLOWLIST_CAP from 5 to 4 (research entry already removed)
- Add test_known_non_conforming_entries_have_tracking_comments to
  enforce that every allowlist entry carries a tracking issue reference
- Update tests/arch/AGENTS.md and resolve-review/SKILL.md registries

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Trecek

src/autoskillit/skills_extended/resolve-review/SKILL.md

@@ -504,7 +504,7 @@ classified `REJECT` with `category: "arch_violation"`.
 | Subagent filter guard | `test_subagent_filter_guard.py` | NDJSON assistant-record consumers missing `_is_parent_assistant_record` or `_is_parent_assistant` predicate — subagent records contaminate parent metrics |
 | Env-var-set constant consumption | `test_canonical_constant_consumption.py` | `*_ENV_FORWARD_VARS` or `*_REQUIRED_ENV` constant with zero production importers — every canonical env-var-set must be consumed |
 | MCP env forward coverage | `test_mcp_env_forward_coverage.py` | `mcp_env_forward_vars` values missing from `CmdSpec.env` in any cmd-builder (skill, food-truck, headless, resume, interactive) |
-| Rule severity consistency | `test_rule_severity_consistency.py` | Direct `RuleFinding()` construction in `@semantic_rule`/`@block_rule` bodies — must use `make_finding()`/`make_block_finding()` |
+| Rule severity consistency | `test_rule_severity_consistency.py` | Direct `RuleFinding()` construction in `@semantic_rule`/`@block_rule` bodies — must use `make_finding()`/`make_block_finding()`; `_KNOWN_NON_CONFORMING_RULES` entries without `# tracking: #NNNN` comments |
 | No direct swap_labels in fleet | `test_swap_labels_guard.py` | Direct `swap_labels` calls in `fleet/` outside `_label_cleanup.py` — must route through `cleanup_orphaned_labels` |
 | Pipeline ordering | `test_pipeline_ordering.py` | Moving `run_semantic_rules` before `_prune_skipped_steps` in `load_and_validate` — semantic rules must run on post-prune recipe |
 

tests/arch/AGENTS.md

@@ -76,7 +76,7 @@ AST enforcement, sub-package layer contracts, and architectural invariant tests.
 | `test_python_no_hardcoded_temp.py` | Architectural invariant: no literal `.autoskillit/temp` outside the whitelist |
 | `test_quota_capability_isolation.py` | AST guard: quota modules must not reference BackendCapabilities fields |
 | `test_recipe_rule_registration.py` | REQ-RECIPE-001: every recipe/rules_*.py file must be imported by recipe/__init__.py |
-| `test_rule_severity_consistency.py` | AST guards: rule functions must use make_finding()/make_block_finding() and RuleFinding severity must match @semantic_rule decorator severity |
+| `test_rule_severity_consistency.py` | AST guards: rule functions must use make_finding()/make_block_finding(); RuleFinding severity must match @semantic_rule decorator severity; _KNOWN_NON_CONFORMING_RULES entries must carry tracking comments |
 | `test_regex_guards.py` | Arch guard: keyword regexes in cmd-scanning rules must use path-safe lookbehind guards |
 | `test_regex_import.py` | Structural guard: src/ must use `import regex as re`, not bare `import re` (hooks/ exempt) |
 | `test_registry.py` | Symbolic rule registry tests (RuleDescriptor, RULES, Violation) |

tests/arch/test_rule_severity_consistency.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import ast
+import re as _re
 from pathlib import Path
 
 import pytest
@@ -156,7 +157,7 @@ def test_rule_findings_match_rule_def_severity() -> None:
     Path(__file__).resolve().parents[1] / "recipe" / "test_bundled_recipes_dispatch_ready.py"
 )
 
-_ALLOWLIST_CAP = 5
+_ALLOWLIST_CAP = 4
 
 
 def _collect_allowlist_rule_names() -> set[str]:
@@ -205,6 +206,39 @@ def test_dispatch_readiness_allowlist_size_cap() -> None:
     )
 
 
+def test_known_non_conforming_entries_have_tracking_comments() -> None:
+    """Every _KNOWN_NON_CONFORMING_RULES entry must reference a tracking issue."""
+    source = _DISPATCH_READY_TEST.read_text()
+    lines = source.splitlines()
+    tree = ast.parse(source)
+
+    missing: list[str] = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.AnnAssign):
+            target = node.target
+            value = node.value
+        elif isinstance(node, ast.Assign):
+            target = node.targets[0] if node.targets else None
+            value = node.value
+        else:
+            continue
+        if not isinstance(target, ast.Name) or target.id != "_KNOWN_NON_CONFORMING_RULES":
+            continue
+        if not isinstance(value, ast.Dict):
+            continue
+        for key in value.keys:
+            if isinstance(key, ast.Constant) and isinstance(key.value, str):
+                line = lines[key.lineno - 1]
+                if not _re.search(r"#\s*tracking:\s*#\d+", line):
+                    missing.append(f"{key.value!r} (line {key.lineno})")
+
+    assert not missing, (
+        "Entries in _KNOWN_NON_CONFORMING_RULES missing tracking comments: "
+        + ", ".join(missing)
+        + ". Add '# tracking: #NNNN' with the relevant GitHub issue number."
+    )
+
+
 def _decorator_rule_name(dec: ast.Call) -> str | None:
     """Extract the ``name=`` keyword string value from a decorator call."""
     for kw in dec.keywords:

tests/recipe/test_bundled_recipes_dispatch_ready.py

@@ -29,11 +29,11 @@
 
 
 _KNOWN_NON_CONFORMING_RULES: dict[str, set[str]] = {
-    "agent-eval": {
+    "agent-eval": {  # tracking: #4069
         "all-dispatchable-stops-have-sentinel",
         "dead-output",
     },
-    "skill-eval": {
+    "skill-eval": {  # tracking: #4069
         "all-dispatchable-stops-have-sentinel",
         "dead-output",
     },