feat: add SARIF 2.1.0 output for code scanning integrations

docs/ci-github-actions.md

@@ -53,6 +53,42 @@ Example CI step:
   run: agent-harness validate "scenarios/**/*.yaml"
 ```
 
+
+## SARIF output for code scanning
+
+`agent-harness run` accepts `--sarif-out <path>` to write assertion results as
+SARIF 2.1.0. Only failed and errored assertions produce SARIF results — passed
+assertions are omitted because SARIF results represent findings, not
+confirmations.
+
+Rule IDs are derived from assertion IDs (`agent-harness/<assertion-id>`) and
+are stable across runs of the same scenario.
+
+```bash
+agent-harness run scenarios/goal_hijack/basic.yaml \
+  --trace-file traces/basic.json \
+  --sarif-out results/basic.sarif
+```
+
+Upload to GitHub Code Scanning:
+
+```yaml
+- name: Run security regression
+  run: |
+    agent-harness run scenarios/goal_hijack/basic.yaml \
+      --trace-file traces/basic.json \
+      --sarif-out results/basic.sarif
+
+- name: Upload SARIF
+  uses: github/codeql-action/upload-sarif@v3
+  with:
+    sarif_file: results/basic.sarif
+  if: always()
+```
+
+**Limitations:** SARIF output reflects regression assertion results, not
+traditional code scanning findings. There are no source locations because
+assertions evaluate runtime agent behaviour rather than static code.
 ## How pass and fail actually work
 
 `agent-harness run` writes machine-readable result JSON to the path you give

src/agent_harness/cli.py

@@ -20,6 +20,7 @@
     run_scenario_with_python_target,
     run_scenario_with_trace,
 )
+from agent_harness.sarif import result_to_sarif
 from agent_harness.scenario import ScenarioValidationError, load_scenario
 from agent_harness.trace import TraceValidationError, load_trace
 
@@ -203,6 +204,10 @@ def build_parser() -> argparse.ArgumentParser:
         type=int,
         help="Optional max_turns value passed to the OpenAI Agents SDK runner.",
     )
+    run_parser.add_argument(
+        "--sarif-out",
+        help="Optional path to write assertion results as SARIF 2.1.0.",
+    )
     run_parser.add_argument(
         "--exit-on-fail",
         action="store_true",
@@ -404,6 +409,9 @@ def main() -> int:
         if args.junit_out:
             Path(args.junit_out).write_text(result_to_junit_xml(result), encoding="utf-8")
 
+        if args.sarif_out:
+            Path(args.sarif_out).write_text(result_to_sarif(result), encoding="utf-8")
+
         if args.exit_on_fail and result.result in {"fail", "error"}:
             return 1
 

src/agent_harness/sarif.py

@@ -0,0 +1,102 @@
+"""SARIF 2.1.0 output for harness results."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from agent_harness.result import HarnessResult
+
+_TOOL_NAME = "agent-harness"
+_TOOL_VERSION = "0.1.0"
+_TOOL_URI = "https://github.com/OWASP/Agent-Security-Regression-Harness"
+_SCHEMA = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json"
+
+_SARIF_LEVEL: dict[str, str] = {
+    "fail": "error",
+    "error": "warning",
+    "pass": "none",
+    "not_run": "none",
+}
+
+
+def result_to_sarif(result: HarnessResult) -> str:
+    """Render a harness result as a SARIF 2.1.0 document.
+
+    Only failed and errored assertions produce SARIF results. Passed and
+    not-run assertions are omitted because SARIF results represent findings,
+    not confirmations. Rule IDs are derived from assertion IDs and are stable
+    across runs of the same scenario.
+    """
+    seen_rule_ids: set[str] = set()
+    rules: list[dict[str, Any]] = []
+    sarif_results: list[dict[str, Any]] = []
+
+    for assertion in result.assertions:
+        if assertion.result in {"pass", "not_run"}:
+            continue
+
+        rule_id = f"agent-harness/{assertion.id}"
+
+        if rule_id not in seen_rule_ids:
+            seen_rule_ids.add(rule_id)
+            rules.append(
+                {
+                    "id": rule_id,
+                    "name": assertion.id,
+                    "shortDescription": {
+                        "text": f"Security assertion: {assertion.id}",
+                    },
+                    "helpUri": _TOOL_URI,
+                    "properties": {
+                        "tags": ["security", "regression"],
+                    },
+                }
+            )
+
+        message_text = (
+            f"Scenario '{result.scenario_id}' assertion '{assertion.id}'"
+            f" {assertion.result}"
+        )
+        if assertion.evidence:
+            message_text += f": {assertion.evidence}"
+
+        sarif_results.append(
+            {
+                "ruleId": rule_id,
+                "level": _SARIF_LEVEL[assertion.result],
+                "message": {"text": message_text},
+                "properties": {
+                    "scenario_id": result.scenario_id,
+                    "assertion_id": assertion.id,
+                    "result": assertion.result,
+                    "evidence": assertion.evidence,
+                },
+                "locations": [],
+            }
+        )
+
+    sarif: dict[str, Any] = {
+        "$schema": _SCHEMA,
+        "version": "2.1.0",
+        "runs": [
+            {
+                "tool": {
+                    "driver": {
+                        "name": _TOOL_NAME,
+                        "version": _TOOL_VERSION,
+                        "informationUri": _TOOL_URI,
+                        "rules": rules,
+                    }
+                },
+                "results": sarif_results,
+                "properties": {
+                    "scenario_id": result.scenario_id,
+                    "mode": result.mode,
+                    "overall_result": result.result,
+                },
+            }
+        ],
+    }
+
+    return json.dumps(sarif, indent=2) + "\n"

tests/test_cli.py

@@ -1501,3 +1501,97 @@ def test_target_timeout_requires_live(capsys, monkeypatch, tmp_path):
         main()
 
     assert "--target-timeout can only be used with --live" in capsys.readouterr().err
+
+
+def test_run_trace_file_writes_sarif(capsys, monkeypatch, tmp_path):
+    scenario_file, trace_file = _write_failing_trace_files(tmp_path)
+    sarif_file = tmp_path / "result.sarif"
+
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "agent-harness",
+            "run",
+            str(scenario_file),
+            "--trace-file",
+            str(trace_file),
+            "--sarif-out",
+            str(sarif_file),
+        ],
+    )
+
+    exit_code = main()
+
+    captured = capsys.readouterr()
+    result = json.loads(captured.out)
+    sarif = json.loads(sarif_file.read_text(encoding="utf-8"))
+
+    assert exit_code == 0
+    assert result["result"] == "fail"
+    assert sarif["version"] == "2.1.0"
+
+    runs = sarif["runs"]
+    assert len(runs) == 1
+
+    run = runs[0]
+    assert run["tool"]["driver"]["name"] == "agent-harness"
+
+    rules = run["tool"]["driver"]["rules"]
+    assert len(rules) == 1
+    assert rules[0]["id"] == "agent-harness/no_denied_tool_call"
+
+    results = run["results"]
+    assert len(results) == 1
+
+    sarif_result = results[0]
+    assert sarif_result["ruleId"] == "agent-harness/no_denied_tool_call"
+    assert sarif_result["level"] == "error"
+    assert "goal_hijack.basic_001" in sarif_result["message"]["text"]
+    assert "no_denied_tool_call" in sarif_result["message"]["text"]
+    assert sarif_result["properties"]["scenario_id"] == "goal_hijack.basic_001"
+    assert sarif_result["properties"]["assertion_id"] == "no_denied_tool_call"
+    assert sarif_result["properties"]["result"] == "fail"
+
+
+def test_run_trace_file_sarif_empty_on_pass(capsys, monkeypatch, tmp_path):
+    scenario_file = tmp_path / "scenario.yaml"
+    trace_file = tmp_path / "trace.json"
+    sarif_file = tmp_path / "result.sarif"
+
+    scenario_file.write_text(VALID_SCENARIO, encoding="utf-8")
+    trace_file.write_text(
+        json.dumps(
+            {
+                "messages": [
+                    {
+                        "role": "assistant",
+                        "tool_calls": [],
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "agent-harness",
+            "run",
+            str(scenario_file),
+            "--trace-file",
+            str(trace_file),
+            "--sarif-out",
+            str(sarif_file),
+        ],
+    )
+
+    exit_code = main()
+
+    sarif = json.loads(sarif_file.read_text(encoding="utf-8"))
+
+    assert exit_code == 0
+    assert sarif["runs"][0]["results"] == []
+    assert sarif["runs"][0]["tool"]["driver"]["rules"] == []