feat: [US-004] - Verify all 20 suites at Grade A

sjarmak · claude · sjarmak · commit fdd980dd7eda · 2026-03-07T03:08:46.000Z
- Add missing test.sh wrappers to 12 org tasks (R.1 critical fix)
- Fix T.10 shared-state issues in 6 tasks:
  - ccx-onboard-search-201/202/203: /tmp/verify.py → mktemp
  - etcd-grpc-api-upgrade-001: /tmp/fallback_patch.diff → mktemp
  - python-http-class-naming-refac-001: /tmp/fallback_patch.diff → mktemp
  - flipt-ecr-auth-oci-fix-001: extract test patch heredoc to test_patch.diff

All 20 suites now at Grade A with zero critical or important failures.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/benchmarks/csb_org_compliance/ccx-compliance-286/tests/test.sh b/benchmarks/csb_org_compliance/ccx-compliance-286/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_compliance/ccx-compliance-292/tests/test.sh b/benchmarks/csb_org_compliance/ccx-compliance-292/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_crossorg/ccx-crossorg-288/tests/test.sh b/benchmarks/csb_org_crossorg/ccx-crossorg-288/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_crossorg/ccx-crossorg-295/tests/test.sh b/benchmarks/csb_org_crossorg/ccx-crossorg-295/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_crossrepo_tracing/ccx-dep-trace-293/tests/test.sh b/benchmarks/csb_org_crossrepo_tracing/ccx-dep-trace-293/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_migration/ccx-migration-289/tests/test.sh b/benchmarks/csb_org_migration/ccx-migration-289/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_migration/ccx-migration-294/tests/test.sh b/benchmarks/csb_org_migration/ccx-migration-294/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-201/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-201/tests/test.sh
@@ -26,7 +26,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT=$(mktemp /tmp/verify_XXXXXX.py)
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -64,5 +65,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
+rm -f "$VERIFY_SCRIPT"
 exit 0
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-202/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-202/tests/test.sh
@@ -26,7 +26,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT=$(mktemp /tmp/verify_XXXXXX.py)
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -64,5 +65,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
+rm -f "$VERIFY_SCRIPT"
 exit 0
diff --git a/benchmarks/csb_org_onboarding/ccx-onboard-search-203/tests/test.sh b/benchmarks/csb_org_onboarding/ccx-onboard-search-203/tests/test.sh
@@ -26,7 +26,8 @@ if [ ! -f "$SOLUTION_FILE" ]; then
     exit 0
 fi
 
-cat > /tmp/verify.py << 'PYEOF'
+VERIFY_SCRIPT=$(mktemp /tmp/verify_XXXXXX.py)
+cat > "$VERIFY_SCRIPT" << 'PYEOF'
 import json, sys, re
 sys.path.insert(0, "/tests")
 from verifiers import SemanticRetrievalQAVerifier
@@ -64,5 +65,6 @@ except Exception as e:
         f.write("0.0")
 PYEOF
 
-python3 /tmp/verify.py 2>&1 | tee /logs/verifier/verify-debug.log
+python3 "$VERIFY_SCRIPT" 2>&1 | tee /logs/verifier/verify-debug.log
+rm -f "$VERIFY_SCRIPT"
 exit 0
diff --git a/benchmarks/csb_org_org/ccx-agentic-290/tests/test.sh b/benchmarks/csb_org_org/ccx-agentic-290/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_platform/ccx-platform-285/tests/test.sh b/benchmarks/csb_org_platform/ccx-platform-285/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_platform/ccx-platform-291/tests/test.sh b/benchmarks/csb_org_platform/ccx-platform-291/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_security/ccx-vuln-remed-287/tests/test.sh b/benchmarks/csb_org_security/ccx-vuln-remed-287/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_org_security/ccx-vuln-remed-296/tests/test.sh b/benchmarks/csb_org_security/ccx-vuln-remed-296/tests/test.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# test.sh — Harbor compatibility wrapper
+# Harbor requires tests/test.sh for task discovery (TaskPaths.is_valid() check).
+# The actual evaluation logic lives in eval.sh (SWE-Factory exit-code-first pattern).
+
+# sg_only_env: restore full repo before verification (no-op for regular runs)
+[ -f /tmp/.sg_only_mode ] && [ -f /tests/sgonly_verifier_wrapper.sh ] && source /tests/sgonly_verifier_wrapper.sh
+
+exec bash "$(dirname "$0")/eval.sh" "$@"
diff --git a/benchmarks/csb_sdlc_design/etcd-grpc-api-upgrade-001/tests/test.sh b/benchmarks/csb_sdlc_design/etcd-grpc-api-upgrade-001/tests/test.sh
@@ -32,7 +32,7 @@ else
     echo "No patch at $PATCH_FILE — trying fallback diff collection..." >&2
 
     # Fallback: generate diff from git changes in the source repos
-    FALLBACK_PATCH="/tmp/fallback_patch.diff"
+    FALLBACK_PATCH=$(mktemp /tmp/fallback_XXXXXX.diff)
     > "$FALLBACK_PATCH"
 
     for repo_dir in "$CORPUS_ROOT"/src/*/; do
diff --git a/benchmarks/csb_sdlc_fix/flipt-ecr-auth-oci-fix-001/tests/test.sh b/benchmarks/csb_sdlc_fix/flipt-ecr-auth-oci-fix-001/tests/test.sh
diff --git a/benchmarks/csb_sdlc_fix/flipt-ecr-auth-oci-fix-001/tests/test_patch.diff b/benchmarks/csb_sdlc_fix/flipt-ecr-auth-oci-fix-001/tests/test_patch.diff
diff --git a/benchmarks/csb_sdlc_refactor/python-http-class-naming-refac-001/tests/test.sh b/benchmarks/csb_sdlc_refactor/python-http-class-naming-refac-001/tests/test.sh
