Apply yapf to .claude python files

CI's PR sanity check (yapf) flagged 8 reformatting changes in the new
skill-eval pipeline. Reformat in place using the repo's
`scripts/run_yapf_format.sh` rules (`based_on_style = google`).

Also gitignore `.claude/evals/workspaces/` so per-iteration eval
artifacts (`responses.json`, `grading.*.json`, generated reports) don't
land in commits.

No functional changes; pipeline smoke-tested end-to-end after reformat
(preflight, import_smoke, pick_workflow, runner, programmatic grader,
aggregate, viewer).

Signed-off-by: kvmto <kmato@nvidia.com>

Author: kvmto

.claude/evals/aggregate.py

@@ -113,9 +113,8 @@ def _benchmark(configs: dict[str, dict[str, dict]]) -> dict[str, Any]:
             cfg_out[grader] = {
                 "scenarios_total": len(scenarios),
                 "scenarios_considered": len(considered),
-                "pass_rate": (
-                    sum(considered) / len(considered) if considered else 0.0
-                ),
+                "pass_rate":
+                    (sum(considered) / len(considered) if considered else 0.0),
                 "raw": payload,
             }
         bench["configurations"][cfg_name] = cfg_out
@@ -140,25 +139,35 @@ def _agreement(configs: dict[str, dict[str, dict]]) -> dict[str, Any]:
 
     grader_verdicts: dict[str, list[bool | None]] = {}
     for grader, payload in target.items():
-        verdicts = [_grader_pass(grader, s) for s in payload.get("scenarios", [])]
+        verdicts = [
+            _grader_pass(grader, s) for s in payload.get("scenarios", [])
+        ]
         grader_verdicts[grader] = verdicts
 
     pairs: dict[str, dict[str, Any]] = {}
     for a, b in combinations(grader_verdicts, 2):
-        pairs[f"{a} ↔ {b}"] = _cohen_kappa(grader_verdicts[a], grader_verdicts[b])
+        pairs[f"{a} ↔ {b}"] = _cohen_kappa(grader_verdicts[a],
+                                           grader_verdicts[b])
     return {
-        "configuration": "with_skill" if "with_skill" in configs else next(iter(configs)),
-        "pairwise_kappa": pairs,
+        "configuration":
+            "with_skill" if "with_skill" in configs else next(iter(configs)),
+        "pairwise_kappa":
+            pairs,
     }
 
 
 def main() -> int:
-    p = argparse.ArgumentParser(description=__doc__,
-                                formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("iteration_dir", type=Path,
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("iteration_dir",
+                   type=Path,
                    help="Path to the iteration directory.")
-    p.add_argument("--out", type=Path, default=None,
-                   help="Output benchmark.json path. Default: <iteration>/benchmark.json")
+    p.add_argument(
+        "--out",
+        type=Path,
+        default=None,
+        help="Output benchmark.json path. Default: <iteration>/benchmark.json")
     args = p.parse_args()
 
     if not args.iteration_dir.is_dir():
@@ -190,8 +199,11 @@ def main() -> int:
             print(f"  {g:<14} {d:+.0%}")
     if bench["inter_grader_agreement"].get("pairwise_kappa"):
         print()
-        print(f"Inter-grader Cohen's κ ({bench['inter_grader_agreement']['configuration']}):")
-        for pair, k in bench["inter_grader_agreement"]["pairwise_kappa"].items():
+        print(
+            f"Inter-grader Cohen's κ ({bench['inter_grader_agreement']['configuration']}):"
+        )
+        for pair, k in bench["inter_grader_agreement"]["pairwise_kappa"].items(
+        ):
             kv = k.get("kappa")
             ks = "n/a" if kv is None else f"{kv:+.2f}"
             print(f"  {pair:<32}  κ = {ks}  (n={k['n']})")

.claude/evals/graders/executable.py

@@ -72,17 +72,21 @@ def _resolve_assertions(skill: str, override: Path | None) -> Path:
     if override is not None:
         return override
     if skill not in SKILL_DIRS:
-        raise SystemExit(f"Unknown skill alias '{skill}'. Known: {sorted(SKILL_DIRS)}")
+        raise SystemExit(
+            f"Unknown skill alias '{skill}'. Known: {sorted(SKILL_DIRS)}")
     p = ASSERTIONS_DIR / f"{SKILL_DIRS[skill]}.json"
     if not p.exists():
         raise SystemExit(f"Assertions not found at {p}")
     return p
 
 
 _FENCE_PATTERNS = {
-    "first_python_block": re.compile(r"```(?:python|py)\s*\n(.*?)\n```", re.DOTALL),
-    "first_bash_block":   re.compile(r"```(?:bash|sh|shell)\s*\n(.*?)\n```", re.DOTALL),
-    "all_python_blocks":  re.compile(r"```(?:python|py)\s*\n(.*?)\n```", re.DOTALL),
+    "first_python_block":
+        re.compile(r"```(?:python|py)\s*\n(.*?)\n```", re.DOTALL),
+    "first_bash_block":
+        re.compile(r"```(?:bash|sh|shell)\s*\n(.*?)\n```", re.DOTALL),
+    "all_python_blocks":
+        re.compile(r"```(?:python|py)\s*\n(.*?)\n```", re.DOTALL),
 }
 
 
@@ -108,30 +112,45 @@ def _module_present(mod: str) -> bool:
 def _run_scenario(scenario_id: str, spec: dict, response: str) -> dict:
     rules = spec.get("executable")
     if not rules:
-        return {"id": scenario_id, "status": "skipped",
-                "reason": "no executable block in assertions"}
+        return {
+            "id": scenario_id,
+            "status": "skipped",
+            "reason": "no executable block in assertions"
+        }
 
     # Skip if a required module is missing (saves time + noise).
     for needed in rules.get("skip_if_missing", []):
         if not _module_present(needed):
-            return {"id": scenario_id, "status": "skipped",
-                    "reason": f"required module '{needed}' not installed"}
+            return {
+                "id": scenario_id,
+                "status": "skipped",
+                "reason": f"required module '{needed}' not installed"
+            }
 
     extractor = rules.get("code_extractor", "first_python_block")
     code = _extract_code(response, extractor)
     if code is None:
-        return {"id": scenario_id, "status": "no_code",
-                "reason": f"no `{extractor}` block found in response"}
+        return {
+            "id": scenario_id,
+            "status": "no_code",
+            "reason": f"no `{extractor}` block found in response"
+        }
 
-    interp = rules.get("interpreter", "python3" if "python" in extractor else "bash")
+    interp = rules.get("interpreter",
+                       "python3" if "python" in extractor else "bash")
     preamble = rules.get("preamble", "")
     harness = rules.get("harness", "{code}")
-    program = harness.format(code=preamble + code) if "{code}" in harness else preamble + code
+    program = harness.format(code=preamble +
+                             code) if "{code}" in harness else preamble + code
     timeout_s = float(rules.get("timeout_s", 30))
 
-    cmd = [interp, "-c", program] if interp != "bash" else ["bash", "-c", program]
+    cmd = [interp, "-c", program
+          ] if interp != "bash" else ["bash", "-c", program]
     try:
-        r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_s)
+        r = subprocess.run(cmd,
+                           capture_output=True,
+                           text=True,
+                           timeout=timeout_s)
         rc, out, err, timed_out = r.returncode, r.stdout, r.stderr, False
     except subprocess.TimeoutExpired:
         rc, out, err, timed_out = 124, "", f"timeout after {timeout_s}s", True
@@ -167,7 +186,9 @@ def grade(skill: str, responses: dict, assertions_path: Path) -> dict:
     for sid, spec in bench["scenarios"].items():
         scenarios.append(_run_scenario(sid, spec, responses.get(sid, "")))
 
-    counted = [s for s in scenarios if s["status"] in {"passed", "failed", "no_code"}]
+    counted = [
+        s for s in scenarios if s["status"] in {"passed", "failed", "no_code"}
+    ]
     passed = sum(1 for s in scenarios if s["status"] == "passed")
     failed = sum(1 for s in scenarios if s["status"] == "failed")
     no_code = sum(1 for s in scenarios if s["status"] == "no_code")
@@ -188,9 +209,11 @@ def grade(skill: str, responses: dict, assertions_path: Path) -> dict:
 
 
 def main() -> int:
-    p = argparse.ArgumentParser(description=__doc__,
-                                formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("--skill", required=True,
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--skill",
+                   required=True,
                    help=f"Skill alias. Known: {sorted(SKILL_DIRS)}")
     p.add_argument("--responses", type=Path, required=True)
     p.add_argument("--assertions", type=Path, default=None)
@@ -216,10 +239,14 @@ def main() -> int:
         print(f"  Failed:      {result['failed']}")
         print(f"  No code:     {result['no_code']}")
         print(f"  Skipped:     {result['skipped']}")
-        print(f"  Pass rate:   {result['scenario_pass_rate']:.0%} (over passed+failed+no_code)")
+        print(
+            f"  Pass rate:   {result['scenario_pass_rate']:.0%} (over passed+failed+no_code)"
+        )
         for s in result["scenarios"]:
             if s["status"] == "failed":
-                print(f"  [{s['id']}] FAILED  exit={s['exit_code']}/{s['expected_exit']}")
+                print(
+                    f"  [{s['id']}] FAILED  exit={s['exit_code']}/{s['expected_exit']}"
+                )
                 print(f"    stderr_tail: {s['stderr_tail'][:200]}")
         print(f"  Wrote: {out}")
     return 0

.claude/evals/graders/judge.py

@@ -66,9 +66,11 @@
 }
 
 
-def _resolve_skill_files(skill: str, ass_override: Path | None) -> tuple[Path, Path]:
+def _resolve_skill_files(skill: str,
+                         ass_override: Path | None) -> tuple[Path, Path]:
     if skill not in SKILL_DIRS:
-        raise SystemExit(f"Unknown skill alias '{skill}'. Known: {sorted(SKILL_DIRS)}")
+        raise SystemExit(
+            f"Unknown skill alias '{skill}'. Known: {sorted(SKILL_DIRS)}")
     full = SKILL_DIRS[skill]
     a = ass_override or (ASSERTIONS_DIR / f"{full}.json")
     p = PROMPTS_DIR / f"{full}.evals.json"
@@ -113,7 +115,6 @@ def _resolve_skill_files(skill: str, ass_override: Path | None) -> tuple[Path, P
 must_not_include: {must_not_include}
 """
 
-
 # ---------------------------------------------------------------------------
 # Backends
 # ---------------------------------------------------------------------------
@@ -131,7 +132,10 @@ def _backend_openai(model: str) -> Callable[[str], str] | None:
     def call(prompt: str) -> str:
         resp = client.chat.completions.create(
             model=model,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{
+                "role": "user",
+                "content": prompt
+            }],
             temperature=0,
         )
         return resp.choices[0].message.content or ""
@@ -153,7 +157,10 @@ def call(prompt: str) -> str:
             model=model,
             max_tokens=400,
             temperature=0,
-            messages=[{"role": "user", "content": prompt}],
+            messages=[{
+                "role": "user",
+                "content": prompt
+            }],
         )
         text = ""
         for block in resp.content:
@@ -186,7 +193,8 @@ def _parse_judge_reply(text: str) -> dict[str, Any] | None:
         except (TypeError, ValueError):
             d[k] = 0
     d.setdefault("justification", "")
-    d["total"] = sum(d[k] for k in ("correctness", "specificity", "coverage", "hallucinations"))
+    d["total"] = sum(d[k] for k in ("correctness", "specificity", "coverage",
+                                    "hallucinations"))
     return d
 
 
@@ -199,11 +207,12 @@ def grade(skill: str, prompts: list[dict], assertions: dict,
         spec = assertions["scenarios"].get(sid)
         if not spec:
             continue
-        rendered = (RUBRIC
-                    .replace("{prompt}", entry["prompt"])
-                    .replace("{response}", responses.get(sid, "(no response)"))
-                    .replace("{must_include}", json.dumps(spec.get("must_include", [])))
-                    .replace("{must_not_include}", json.dumps(spec.get("must_not_include", []))))
+        rendered = (RUBRIC.replace("{prompt}", entry["prompt"]).replace(
+            "{response}", responses.get(sid, "(no response)")).replace(
+                "{must_include}",
+                json.dumps(spec.get("must_include", []))).replace(
+                    "{must_not_include}",
+                    json.dumps(spec.get("must_not_include", []))))
 
         record = {"id": sid, "model": model}
         if call is None:
@@ -230,8 +239,8 @@ def grade(skill: str, prompts: list[dict], assertions: dict,
         scored.append(record)
 
     have_scores = [r for r in scored if r["status"] == "scored"]
-    avg = (sum(r["total"] for r in have_scores) / (len(have_scores) * 8)
-           if have_scores else 0.0)
+    avg = (sum(r["total"] for r in have_scores) /
+           (len(have_scores) * 8) if have_scores else 0.0)
 
     return {
         "grader": "judge",
@@ -245,19 +254,26 @@ def grade(skill: str, prompts: list[dict], assertions: dict,
 
 
 def main() -> int:
-    p = argparse.ArgumentParser(description=__doc__,
-                                formatter_class=argparse.RawDescriptionHelpFormatter)
-    p.add_argument("--skill", required=True, help=f"One of: {sorted(SKILL_DIRS)}")
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--skill",
+                   required=True,
+                   help=f"One of: {sorted(SKILL_DIRS)}")
     p.add_argument("--responses", type=Path, required=True)
     p.add_argument("--assertions", type=Path, default=None)
     p.add_argument("--backend", choices=sorted(BACKENDS), default="anthropic")
-    p.add_argument("--model", default="claude-opus-4-5",
-                   help="Pin a specific model. Recorded in the output for reproducibility.")
+    p.add_argument(
+        "--model",
+        default="claude-opus-4-5",
+        help="Pin a specific model. Recorded in the output for reproducibility."
+    )
     p.add_argument("--out", type=Path, default=None)
     p.add_argument("--quiet", action="store_true")
     args = p.parse_args()
 
-    prompts_path, assertions_path = _resolve_skill_files(args.skill, args.assertions)
+    prompts_path, assertions_path = _resolve_skill_files(
+        args.skill, args.assertions)
     if not args.responses.exists():
         raise SystemExit(f"Responses file not found: {args.responses}")
     responses = json.loads(args.responses.read_text())
@@ -274,8 +290,12 @@ def main() -> int:
 
     if not args.quiet:
         print(f"Skill: {result['skill']}  judge={args.backend}/{args.model}")
-        print(f"  Scored: {result['scenarios_scored']}/{result['scenarios_total']}")
-        print(f"  Avg normalized score (0-1): {result['average_normalized_score']:.2f}")
+        print(
+            f"  Scored: {result['scenarios_scored']}/{result['scenarios_total']}"
+        )
+        print(
+            f"  Avg normalized score (0-1): {result['average_normalized_score']:.2f}"
+        )
         if result["scenarios_scored"] == 0:
             print("  (no scenarios scored — check 'reason' fields in the JSON)")
         print(f"  Wrote: {out}")