Add hosted model baseline runners

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: trynullsec

.env.example

@@ -4,3 +4,11 @@
 NULLSEC_ADAPTER_PATH=outputs/nullsec-s1-qlora
 NULLSEC_BASE_MODEL=Qwen/Qwen2.5-Coder-7B-Instruct
 NULLSEC_MAX_NEW_TOKENS=1536
+
+# Optional hosted-model baseline evaluation.
+# No defaults are provided because exact provider model IDs must be recorded in
+# generated reports.
+ANTHROPIC_API_KEY=
+ANTHROPIC_MODEL=
+OPENAI_API_KEY=
+OPENAI_MODEL=

benchmarks/baselines/api_common.py

@@ -0,0 +1,95 @@
+"""Shared helpers for hosted API model baselines.
+
+API baselines are comparison-only. They use the same prompt and scoring pipeline
+as Nullsec-S1, write raw responses incrementally for resumability, and never
+invent metrics.
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+from benchmarks.baselines.common import write_baseline_report
+from benchmarks.harness import finalize_raw, load_dataset
+from nullsec.core.prompts import ANALYZE_TEMPLATE, SYSTEM_PROMPT
+
+REPORT_ROOT = Path(__file__).resolve().parents[2] / "benchmarks" / "reports" / "baselines"
+
+
+def user_prompt(case: dict) -> str:
+    return ANALYZE_TEMPLATE.format(
+        filename=case.get("filename", "input"),
+        lang=case.get("lang", ""),
+        code=case["code"],
+    )
+
+
+def load_raw_cache(path: Path) -> dict[str, dict]:
+    if not path.exists():
+        return {}
+    out: dict[str, dict] = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if not line.strip():
+            continue
+        obj = json.loads(line)
+        out[obj["id"]] = obj
+    return out
+
+
+def append_raw(path: Path, obj: dict) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(obj) + "\n")
+
+
+def run_api_baseline(
+    *,
+    provider_id: str,
+    system_name: str,
+    dataset_name: str,
+    model: str,
+    limit: int | None,
+    resume: bool,
+    sleep_s: float,
+    call_model,
+    provenance: dict,
+) -> Path:
+    ds = load_dataset(dataset_name)
+    cases = ds["cases"][:limit] if limit else ds["cases"]
+    out_dir = REPORT_ROOT / provider_id
+    raw_path = out_dir / "raw_outputs.jsonl"
+    cached = load_raw_cache(raw_path) if resume else {}
+    verdicts: dict = {}
+
+    print(f"{provider_id}: {len(cases)} case(s), model={model}, resume={resume}")
+    for i, case in enumerate(cases, start=1):
+        cid = case["id"]
+        if cid in cached:
+            raw = cached[cid]["raw"]
+            print(f"[{i}/{len(cases)}] {cid}: cached")
+        else:
+            print(f"[{i}/{len(cases)}] {cid}: requesting")
+            raw = call_model(SYSTEM_PROMPT, user_prompt(case))
+            append_raw(raw_path, {"id": cid, "raw": raw, "provider": provider_id, "model": model})
+            if sleep_s:
+                time.sleep(sleep_s)
+        verdicts[cid] = finalize_raw(raw)
+
+    report = write_baseline_report(
+        system_id=provider_id,
+        system_name=system_name,
+        dataset=dataset_name,
+        cases=cases,
+        verdicts=verdicts,
+        provenance={
+            "provider": provider_id,
+            "model": model,
+            "limit": limit,
+            "raw_outputs": str(raw_path),
+            **provenance,
+        },
+    )
+    print(f"{provider_id} baseline report -> {report}")
+    print("Generated report/cache are under benchmarks/reports/ and should not be committed unless explicitly approved.")
+    return report

benchmarks/baselines/claude_api.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Claude API baseline for the Nullsec-S1 benchmark.
+
+Requires ANTHROPIC_API_KEY and ANTHROPIC_MODEL (or --model). Generated reports
+and raw outputs are written under benchmarks/reports/baselines/claude/ and are
+not committed by default.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from benchmarks.baselines.api_common import run_api_baseline
+
+
+def require_env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise SystemExit(f"{name} is required. Export {name}=... before running this baseline.")
+    return value
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Claude API baseline for the Nullsec-S1 benchmark")
+    ap.add_argument("--dataset", default="detection.json")
+    ap.add_argument("--limit", type=int, default=None, help="run only the first N cases (smoke test)")
+    ap.add_argument("--model", default=os.environ.get("ANTHROPIC_MODEL"))
+    ap.add_argument("--resume", action="store_true", help="reuse cached raw_outputs.jsonl entries")
+    ap.add_argument("--sleep", type=float, default=0.0, help="seconds to sleep between requests")
+    ap.add_argument("--max-tokens", type=int, default=1536)
+    args = ap.parse_args()
+
+    api_key = require_env("ANTHROPIC_API_KEY")
+    if not args.model:
+        raise SystemExit("ANTHROPIC_MODEL is required (or pass --model).")
+
+    try:
+        import anthropic
+    except ImportError as e:
+        raise SystemExit("anthropic package is required. Install with: python -m pip install anthropic") from e
+
+    client = anthropic.Anthropic(api_key=api_key)
+
+    def call(system_prompt: str, prompt: str) -> str:
+        msg = client.messages.create(
+            model=args.model,
+            max_tokens=args.max_tokens,
+            temperature=0,
+            system=system_prompt,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        parts = []
+        for block in msg.content:
+            if getattr(block, "type", None) == "text":
+                parts.append(block.text)
+        return "\n".join(parts).strip()
+
+    run_api_baseline(
+        provider_id="claude",
+        system_name=f"Claude API baseline ({args.model})",
+        dataset_name=args.dataset,
+        model=args.model,
+        limit=args.limit,
+        resume=args.resume,
+        sleep_s=args.sleep,
+        call_model=call,
+        provenance={"api": "anthropic_messages", "max_tokens": args.max_tokens},
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/baselines/openai_api.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""OpenAI/Codex API baseline for the Nullsec-S1 benchmark.
+
+Requires OPENAI_API_KEY and OPENAI_MODEL (or --model). Generated reports and raw
+outputs are written under benchmarks/reports/baselines/openai/ and are not
+committed by default.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from benchmarks.baselines.api_common import run_api_baseline
+
+
+def require_env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise SystemExit(f"{name} is required. Export {name}=... before running this baseline.")
+    return value
+
+
+def _extract_response_text(resp) -> str:
+    text = getattr(resp, "output_text", None)
+    if text:
+        return text.strip()
+    chunks = []
+    for item in getattr(resp, "output", []) or []:
+        for content in getattr(item, "content", []) or []:
+            if getattr(content, "type", None) in {"output_text", "text"}:
+                chunks.append(getattr(content, "text", ""))
+    return "\n".join(chunks).strip()
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="OpenAI/Codex API baseline for the Nullsec-S1 benchmark")
+    ap.add_argument("--dataset", default="detection.json")
+    ap.add_argument("--limit", type=int, default=None, help="run only the first N cases (smoke test)")
+    ap.add_argument("--model", default=os.environ.get("OPENAI_MODEL"))
+    ap.add_argument("--resume", action="store_true", help="reuse cached raw_outputs.jsonl entries")
+    ap.add_argument("--sleep", type=float, default=0.0, help="seconds to sleep between requests")
+    ap.add_argument("--max-tokens", type=int, default=1536)
+    args = ap.parse_args()
+
+    api_key = require_env("OPENAI_API_KEY")
+    if not args.model:
+        raise SystemExit("OPENAI_MODEL is required (or pass --model).")
+
+    try:
+        from openai import OpenAI
+    except ImportError as e:
+        raise SystemExit("openai package is required. Install with: python -m pip install openai") from e
+
+    client = OpenAI(api_key=api_key)
+
+    def call(system_prompt: str, prompt: str) -> str:
+        try:
+            resp = client.responses.create(
+                model=args.model,
+                temperature=0,
+                max_output_tokens=args.max_tokens,
+                input=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": prompt},
+                ],
+            )
+            return _extract_response_text(resp)
+        except AttributeError:
+            # Compatibility with older SDKs that only expose chat.completions.
+            resp = client.chat.completions.create(
+                model=args.model,
+                temperature=0,
+                max_tokens=args.max_tokens,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": prompt},
+                ],
+            )
+            return resp.choices[0].message.content.strip()
+
+    run_api_baseline(
+        provider_id="openai",
+        system_name=f"OpenAI/Codex API baseline ({args.model})",
+        dataset_name=args.dataset,
+        model=args.model,
+        limit=args.limit,
+        resume=args.resume,
+        sleep_s=args.sleep,
+        call_model=call,
+        provenance={"api": "openai_responses_or_chat", "max_tokens": args.max_tokens},
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

benchmarks/compare_baselines.py

@@ -69,6 +69,8 @@ def main() -> int:
     ap.add_argument("--nullsec", help="Nullsec-S1 SUITE.json")
     ap.add_argument("--base", help="base Qwen SUITE.json")
     ap.add_argument("--semgrep", help="Semgrep SUITE.json")
+    ap.add_argument("--claude", help="Claude API baseline SUITE.json")
+    ap.add_argument("--openai", help="OpenAI/Codex API baseline SUITE.json")
     ap.add_argument("--out", default=None, help="optional Markdown output path")
     args = ap.parse_args()
 
@@ -77,6 +79,10 @@ def main() -> int:
         row("Base Qwen", load_report(args.base, "base Qwen"), "base model, no Nullsec adapter"),
         row("Semgrep", load_report(args.semgrep, "Semgrep"), "static rules; partial category coverage"),
     ]
+    if args.claude:
+        reports.append(row("Claude", load_report(args.claude, "Claude"), "hosted API baseline; model id/date in report"))
+    if args.openai:
+        reports.append(row("OpenAI/Codex", load_report(args.openai, "OpenAI/Codex"), "hosted API baseline; model id/date in report"))
     md = render(reports)
     if args.out:
         p = Path(args.out)

docs/EVALS.md

@@ -84,6 +84,65 @@ specialized rules). Unsupported categories are documented in the report.
 
 Docker fallback for Semgrep is a future enhancement, not implemented today.
 
+### Claude API
+
+Claude comparisons are optional hosted-model baselines. They require an API key
+and an explicit model id; no default model is hardcoded because provider model
+IDs and dates must be recorded in the report.
+
+Smoke test:
+
+```bash
+export ANTHROPIC_API_KEY=...
+export ANTHROPIC_MODEL=...
+python benchmarks/baselines/claude_api.py --limit 5 --sleep 1
+```
+
+Full run (costs money; run intentionally):
+
+```bash
+python benchmarks/baselines/claude_api.py --sleep 1
+```
+
+Report and raw cache:
+
+```text
+benchmarks/reports/baselines/claude/SUITE.json
+benchmarks/reports/baselines/claude/raw_outputs.jsonl
+```
+
+Use `--resume` to skip already-cached case ids if a run is interrupted.
+
+### OpenAI / Codex API
+
+OpenAI/Codex comparisons are optional hosted-model baselines. They require an API
+key and an explicit model id via `OPENAI_MODEL` or `--model`.
+
+Smoke test:
+
+```bash
+export OPENAI_API_KEY=...
+export OPENAI_MODEL=...
+python benchmarks/baselines/openai_api.py --limit 5 --sleep 1
+```
+
+Full run (costs money; run intentionally):
+
+```bash
+python benchmarks/baselines/openai_api.py --sleep 1
+```
+
+Report and raw cache:
+
+```text
+benchmarks/reports/baselines/openai/SUITE.json
+benchmarks/reports/baselines/openai/raw_outputs.jsonl
+```
+
+Provider models can change over time. Reports record the exact provider, model
+id, run date, dataset, and raw-output cache path. Do not compare hosted-model
+results without those fields.
+
 ## Comparison table
 
 Generate a Markdown comparison from existing reports:
@@ -93,6 +152,8 @@ python benchmarks/compare_baselines.py \
   --nullsec benchmarks/reports/SUITE.json \
   --base benchmarks/reports/baselines/qwen2_5_coder_7b/SUITE.json \
   --semgrep benchmarks/reports/baselines/semgrep/SUITE.json \
+  --claude benchmarks/reports/baselines/claude/SUITE.json \
+  --openai benchmarks/reports/baselines/openai/SUITE.json \
   --out benchmarks/reports/baselines/COMPARISON.md
 ```
 
@@ -140,7 +201,7 @@ output was not alignable for scoring, so the comparison table shows `110`.
 - Semgrep is not expected to cover all categories and should be interpreted as a
   static-analysis baseline, not a security LLM.
 - Frontier/API model baselines such as Claude, GPT, or other hosted models are
-  not included yet.
+  optional and must be generated from scripts with exact model IDs recorded.
 - This comparison does not prove universal vulnerability detection performance.
 - Do not claim Nullsec-S1 beats another model or tool unless the comparison
   script output proves it.