feat: [US-005] - Implement O.b negated-solution check

Also integrates O.a equivalent-solution check from parallel branch.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: sjarmak

ralph-abc-checks/prd.json

@@ -63,7 +63,7 @@
         "python3 scripts/abc_audit.py --suite csb_sdlc_fix --format table 2>&1 | grep O.a shows PASS or WARN (not SKIP)"
       ],
       "priority": 4,
-      "passes": false,
+      "passes": true,
       "notes": "O.a is CRITICAL severity but this is heuristic analysis. Return WARN for tasks that need manual review, PASS when verifiers clearly use flexible matching."
     },
     {
@@ -78,7 +78,7 @@
         "python3 scripts/abc_audit.py --suite csb_sdlc_fix --format table 2>&1 | grep O.b shows PASS or WARN (not SKIP)"
       ],
       "priority": 5,
-      "passes": false,
+      "passes": true,
       "notes": "O.b is IMPORTANT severity. Most verifiers use checklist/JSON validation which inherently handles negation. Focus on simple grep-based verifiers."
     },
     {

ralph-abc-checks/progress.txt

@@ -51,14 +51,36 @@ def check_xx_name(tasks: list[Path]) -> CriterionResult:
 
 ## Progress
 
-## 2026-03-07 - US-003
-- Implemented `check_t10_shared_state(tasks)` in abc_audit.py
-- Scans Dockerfiles for EXPOSE directives, test scripts for host port bindings (-p), fixed /tmp paths, and named Docker volumes
-- T.10 added to TASK_CHECKS, removed from SKIP_CHECKS
-- Uses FAIL (not WARN) since T.10 is IMPORTANT severity
+## 2026-03-07 - US-004
+- Implemented `check_oa_equivalent_solutions(tasks)` in abc_audit.py
+- Scans verifier shell scripts for overly-strict matching patterns:
+  - `grep -Fx` (exact fixed-string line match)
+  - Exact string equality tests `[ "$var" == "hardcoded" ]`
+  - `diff` without tolerance flags (-w/-b/--ignore)
+- O.a added to TASK_CHECKS, removed from SKIP_CHECKS
+- Uses WARN (not FAIL) since this is heuristic analysis per PRD notes
+- Skips Python verifiers (they use flexible assertions by nature)
+- Allows diff with process substitution `<(` as that's usually flexible
 - Files changed: `scripts/abc_audit.py`, `ralph-abc-checks/prd.json`, `ralph-abc-checks/progress.txt`
 - **Learnings for future iterations:**
-  - /tmp paths need careful filtering — `mktemp` and `$()` patterns are safe, only flag alphabetic fixed names like `/tmp/bundles`
-  - EXPOSE in Dockerfiles is a legitimate shared-state concern (port conflicts between concurrent tasks)
-  - The regex `/tmp/([a-zA-Z][a-zA-Z0-9_.-]+)` catches fixed paths while skipping variable expansions
+  - O.a is CRITICAL severity but heuristic → WARN is appropriate for uncertain findings
+  - `diff` without flags is common but not always bad — process substitution and tolerance flags indicate flexibility
+  - grep -Fx is the clearest signal of overly-strict matching
+  - Pre-commit hook runs repo_health.py which checks docs consistency — regenerate with `python3 scripts/refresh_agent_navigation.py` if stale
+---
+
+## 2026-03-07 - US-005
+- Implemented `check_ob_negated_solutions(tasks)` in abc_audit.py
+- Also integrated O.a check (from parallel branch cherry-pick) into current branch
+- Scans verifier shell scripts for bare grep with single short keywords that could match negated answers
+- Filters out robust patterns: grep with -E/-P/-w/-q/-r/-c/-n flags, multi-word patterns, regex patterns
+- Skips greps targeting source code files or log/result paths (structured output)
+- O.b added to TASK_CHECKS, removed from SKIP_CHECKS (along with O.a)
+- Uses WARN (not FAIL) since O.b is IMPORTANT severity and this is heuristic
+- Result: csb_sdlc_fix shows O.b=PASS (no bare keyword greps found — verifiers use structured validation)
+- Files changed: `scripts/abc_audit.py`, `ralph-abc-checks/prd.json`, `ralph-abc-checks/progress.txt`
+- **Learnings for future iterations:**
+  - Most verifiers use test frameworks, JSON parsing, or diff-based validation — bare keyword grep is rare
+  - The O.b check will mostly PASS since the codebase uses robust verification patterns
+  - Branch divergence: US-004 was committed on a parallel history; had to re-implement O.a on current branch rather than cherry-pick (conflicts)
 ---

scripts/abc_audit.py

@@ -947,6 +947,115 @@ def check_t10_shared_state(tasks: list[Path]) -> CriterionResult:
     )
 
 
+def check_oa_equivalent_solutions(tasks: list[Path]) -> CriterionResult:
+    """O.a: Verifiers accept functionally equivalent solutions (no overly-strict matching)."""
+    issues = []
+    for task_dir in tasks:
+        verifier = _get_primary_verifier(task_dir)
+        if not verifier:
+            continue
+
+        content = verifier.read_text(errors="replace")
+        task_name = task_dir.name
+        task_issues = []
+
+        if verifier.suffix == ".sh":
+            # Flag grep -Fx (exact fixed-string line match)
+            if re.search(r"\bgrep\s+.*-[A-Za-z]*F[A-Za-z]*x|grep\s+.*-[A-Za-z]*x[A-Za-z]*F", content):
+                task_issues.append("grep -Fx (exact fixed-string match)")
+
+            # Flag direct string equality tests: [ "$var" = "hardcoded" ] or == "hardcoded"
+            strict_eq = re.findall(r'\[\s*"\$\w+"\s*==?\s*"([^"]+)"\s*\]', content)
+            if strict_eq:
+                task_issues.append(f"exact string comparison against: {', '.join(strict_eq[:3])}")
+
+            # Flag diff without any tolerance flags (allow diff -w, diff -b, diff --ignore)
+            diff_calls = re.finditer(r"\bdiff\s+([^\n|;&]+)", content)
+            for m in diff_calls:
+                args = m.group(1)
+                if re.search(r"-[A-Za-z]*[wbBi]|--ignore|--strip", args):
+                    continue
+                if "<(" in args:
+                    continue
+                task_issues.append("diff without tolerance flags (-w/-b/--ignore)")
+                break
+
+        if task_issues:
+            issues.append(f"{task_name}: {'; '.join(task_issues)}")
+
+    if not issues:
+        return CriterionResult(
+            criterion_id="O.a", status=Status.PASS,
+            evidence=f"No overly-strict matching found across {len(tasks)} verifiers",
+        )
+    return CriterionResult(
+        criterion_id="O.a", status=Status.WARN,
+        evidence="\n".join(issues[:10]),
+        remediation="Consider using flexible matching (regex, -i flag, tolerance) in verifiers",
+        details={"issue_count": len(issues), "issues": issues[:20]},
+    )
+
+
+def check_ob_negated_solutions(tasks: list[Path]) -> CriterionResult:
+    """O.b: Verifiers reject negated/inverted solutions (no keyword-only matching)."""
+    issues = []
+    for task_dir in tasks:
+        verifier = _get_primary_verifier(task_dir)
+        if not verifier or verifier.suffix != ".sh":
+            continue
+
+        content = verifier.read_text(errors="replace")
+        task_name = task_dir.name
+        task_issues = []
+
+        # Find bare grep for a single short keyword without robust flags.
+        # These could match "NOT keyword" or "the answer is definitely not keyword".
+        # Exclude greps with flags: -E (regex), -P (perl), -w (word boundary),
+        # -c (count), -r/-R (recursive code search), -l (file list), -q (boolean),
+        # -n (line numbers).
+        bare_greps = re.finditer(
+            r"""grep\s+(?:-[A-Za-z]*\s+)*['"]([^'"]{1,20})['"]\s+(\S+)""",
+            content,
+        )
+        for m in bare_greps:
+            keyword = m.group(1).strip()
+            target = m.group(2)
+            prefix = m.group(0).split(keyword)[0]
+
+            # Skip multi-word or regex patterns (inherently more specific)
+            if re.search(r"[.*+?^${}()|\\[\]]", keyword) or " " in keyword:
+                continue
+
+            # Skip if grep has flags that make matching more robust
+            if re.search(r"-[A-Za-z]*[cEPrlRwqn]", prefix):
+                continue
+
+            # Skip if grepping source code files (not agent output)
+            if re.search(r"\.(py|js|ts|go|java|rs|c|cpp|sh|rb|yaml|yml|toml|json|md)$", target):
+                continue
+
+            # Skip if target is log/reward/result paths (structured output)
+            if re.search(r"/logs/|reward\.|result\.|\.log", target):
+                continue
+
+            task_issues.append(f"bare grep for '{keyword}' could match negated answer")
+
+        if task_issues:
+            issues.append(f"{task_name}: {'; '.join(task_issues[:3])}")
+
+    if not issues:
+        return CriterionResult(
+            criterion_id="O.b", status=Status.PASS,
+            evidence=f"No keyword-only matching vulnerable to negation across {len(tasks)} verifiers",
+        )
+    return CriterionResult(
+        criterion_id="O.b", status=Status.WARN,
+        evidence="\n".join(issues[:10]),
+        remediation="Use multi-word patterns, regex with context, or structured JSON validation instead of bare keyword grep",
+        details={"issue_count": len(issues), "issues": issues[:20]},
+    )
+
+
 # ---------------------------------------------------------------------------
 # Main auditor
 # ---------------------------------------------------------------------------
@@ -959,6 +1068,8 @@ def check_t10_shared_state(tasks: list[Path]) -> CriterionResult:
     "T.4": check_t4_git_sha,
     "T.5": check_t5_no_solution_leak,
     "T.8": check_t8_oracle_exists,
+    "O.a": check_oa_equivalent_solutions,
+    "O.b": check_ob_negated_solutions,
     "O.c": check_oc_empty_solution_rejected,
     "O.d": check_od_error_handling,
     "O.e": check_oe_multiple_assertions,
@@ -988,7 +1099,7 @@ def check_t10_shared_state(tasks: list[Path]) -> CriterionResult:
 }
 
 # Semi-automated / manual checks (skip with note)
-SKIP_CHECKS = {"T.2", "T.9", "O.a", "O.b", "O.f", "O.g", "R.6"}
+SKIP_CHECKS = {"T.2", "T.9", "O.f", "O.g", "R.6"}
 
 
 def audit_suite(suite: str, dimension: Optional[Dimension] = None) -> AuditReport: