From 0b6385f5f3b1181d537d2ef1fb12724943940b01 Mon Sep 17 00:00:00 2001 From: Omobolaji Adeyan Date: Thu, 18 Jun 2026 13:30:50 -0500 Subject: [PATCH 1/5] feat: add SARIF 2.1.0 output module --- src/agent_harness/sarif.py | 102 +++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 src/agent_harness/sarif.py diff --git a/src/agent_harness/sarif.py b/src/agent_harness/sarif.py new file mode 100644 index 0000000..268f66c --- /dev/null +++ b/src/agent_harness/sarif.py @@ -0,0 +1,102 @@ +"""SARIF 2.1.0 output for harness results.""" + +from __future__ import annotations + +import json +from typing import Any + +from agent_harness.result import HarnessResult + +_TOOL_NAME = "agent-harness" +_TOOL_VERSION = "0.1.0" +_TOOL_URI = "https://github.com/OWASP/Agent-Security-Regression-Harness" +_SCHEMA = "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json" + +_SARIF_LEVEL: dict[str, str] = { + "fail": "error", + "error": "warning", + "pass": "none", + "not_run": "none", +} + + +def result_to_sarif(result: HarnessResult) -> str: + """Render a harness result as a SARIF 2.1.0 document. + + Only failed and errored assertions produce SARIF results. Passed and + not-run assertions are omitted because SARIF results represent findings, + not confirmations. Rule IDs are derived from assertion IDs and are stable + across runs of the same scenario. + """ + seen_rule_ids: set[str] = set() + rules: list[dict[str, Any]] = [] + sarif_results: list[dict[str, Any]] = [] + + for assertion in result.assertions: + if assertion.result in {"pass", "not_run"}: + continue + + rule_id = f"agent-harness/{assertion.id}" + + if rule_id not in seen_rule_ids: + seen_rule_ids.add(rule_id) + rules.append( + { + "id": rule_id, + "name": assertion.id, + "shortDescription": { + "text": f"Security assertion: {assertion.id}", + }, + "helpUri": _TOOL_URI, + "properties": { + "tags": ["security", "regression"], + }, + } + ) + + message_text = ( + f"Scenario '{result.scenario_id}' assertion '{assertion.id}'" + f" {assertion.result}" + ) + if assertion.evidence: + message_text += f": {assertion.evidence}" + + sarif_results.append( + { + "ruleId": rule_id, + "level": _SARIF_LEVEL[assertion.result], + "message": {"text": message_text}, + "properties": { + "scenario_id": result.scenario_id, + "assertion_id": assertion.id, + "result": assertion.result, + "evidence": assertion.evidence, + }, + "locations": [], + } + ) + + sarif: dict[str, Any] = { + "$schema": _SCHEMA, + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": _TOOL_NAME, + "version": _TOOL_VERSION, + "informationUri": _TOOL_URI, + "rules": rules, + } + }, + "results": sarif_results, + "properties": { + "scenario_id": result.scenario_id, + "mode": result.mode, + "overall_result": result.result, + }, + } + ], + } + + return json.dumps(sarif, indent=2) + "\n" \ No newline at end of file From 8029f0d950fb404be68dcb2ed02af78e61564246 Mon Sep 17 00:00:00 2001 From: Omobolaji Adeyan Date: Thu, 18 Jun 2026 13:31:57 -0500 Subject: [PATCH 2/5] feat: add --sarif-out flag to run command --- src/agent_harness/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/agent_harness/cli.py b/src/agent_harness/cli.py index c9c3eee..95fa12b 100644 --- a/src/agent_harness/cli.py +++ b/src/agent_harness/cli.py @@ -10,6 +10,7 @@ from agent_harness.adapters import DEFAULT_HTTP_TIMEOUT_SECONDS, AdapterError from agent_harness.junit import result_to_junit_xml +from agent_harness.sarif import result_to_sarif from agent_harness.runner import ( dry_run_scenario, run_scenario_live, @@ -203,6 +204,10 @@ def build_parser() -> argparse.ArgumentParser: type=int, help="Optional max_turns value passed to the OpenAI Agents SDK runner.", ) + run_parser.add_argument( + "--sarif-out", + help="Optional path to write assertion results as SARIF 2.1.0.", + ) run_parser.add_argument( "--exit-on-fail", action="store_true", @@ -404,6 +409,9 @@ def main() -> int: if args.junit_out: Path(args.junit_out).write_text(result_to_junit_xml(result), encoding="utf-8") + if args.sarif_out: + Path(args.sarif_out).write_text(result_to_sarif(result), encoding="utf-8") + if args.exit_on_fail and result.result in {"fail", "error"}: return 1 From 5f9e122451d3f546f3a2b55a61b595b6d83adf16 Mon Sep 17 00:00:00 2001 From: Omobolaji Adeyan Date: Thu, 18 Jun 2026 13:33:46 -0500 Subject: [PATCH 3/5] test: add SARIF output tests for run command --- tests/test_cli.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 19513a0..af55833 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1501,3 +1501,97 @@ def test_target_timeout_requires_live(capsys, monkeypatch, tmp_path): main() assert "--target-timeout can only be used with --live" in capsys.readouterr().err + + +def test_run_trace_file_writes_sarif(capsys, monkeypatch, tmp_path): + scenario_file, trace_file = _write_failing_trace_files(tmp_path) + sarif_file = tmp_path / "result.sarif" + + monkeypatch.setattr( + sys, + "argv", + [ + "agent-harness", + "run", + str(scenario_file), + "--trace-file", + str(trace_file), + "--sarif-out", + str(sarif_file), + ], + ) + + exit_code = main() + + captured = capsys.readouterr() + result = json.loads(captured.out) + sarif = json.loads(sarif_file.read_text(encoding="utf-8")) + + assert exit_code == 0 + assert result["result"] == "fail" + assert sarif["version"] == "2.1.0" + + runs = sarif["runs"] + assert len(runs) == 1 + + run = runs[0] + assert run["tool"]["driver"]["name"] == "agent-harness" + + rules = run["tool"]["driver"]["rules"] + assert len(rules) == 1 + assert rules[0]["id"] == "agent-harness/no_denied_tool_call" + + results = run["results"] + assert len(results) == 1 + + sarif_result = results[0] + assert sarif_result["ruleId"] == "agent-harness/no_denied_tool_call" + assert sarif_result["level"] == "error" + assert "goal_hijack.basic_001" in sarif_result["message"]["text"] + assert "no_denied_tool_call" in sarif_result["message"]["text"] + assert sarif_result["properties"]["scenario_id"] == "goal_hijack.basic_001" + assert sarif_result["properties"]["assertion_id"] == "no_denied_tool_call" + assert sarif_result["properties"]["result"] == "fail" + + +def test_run_trace_file_sarif_empty_on_pass(capsys, monkeypatch, tmp_path): + scenario_file = tmp_path / "scenario.yaml" + trace_file = tmp_path / "trace.json" + sarif_file = tmp_path / "result.sarif" + + scenario_file.write_text(VALID_SCENARIO, encoding="utf-8") + trace_file.write_text( + json.dumps( + { + "messages": [ + { + "role": "assistant", + "tool_calls": [], + } + ] + } + ), + encoding="utf-8", + ) + + monkeypatch.setattr( + sys, + "argv", + [ + "agent-harness", + "run", + str(scenario_file), + "--trace-file", + str(trace_file), + "--sarif-out", + str(sarif_file), + ], + ) + + exit_code = main() + + sarif = json.loads(sarif_file.read_text(encoding="utf-8")) + + assert exit_code == 0 + assert sarif["runs"][0]["results"] == [] + assert sarif["runs"][0]["tool"]["driver"]["rules"] == [] \ No newline at end of file From 9fc3f5bfd7743a323d514c042579109dd817e730 Mon Sep 17 00:00:00 2001 From: Omobolaji Adeyan Date: Thu, 18 Jun 2026 13:34:38 -0500 Subject: [PATCH 4/5] docs: document --sarif-out flag and GitHub Code Scanning integration --- docs/ci-github-actions.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/ci-github-actions.md b/docs/ci-github-actions.md index 3d7195e..01a2372 100644 --- a/docs/ci-github-actions.md +++ b/docs/ci-github-actions.md @@ -53,6 +53,42 @@ Example CI step: run: agent-harness validate "scenarios/**/*.yaml" ``` + +## SARIF output for code scanning + +`agent-harness run` accepts `--sarif-out ` to write assertion results as +SARIF 2.1.0. Only failed and errored assertions produce SARIF results — passed +assertions are omitted because SARIF results represent findings, not +confirmations. + +Rule IDs are derived from assertion IDs (`agent-harness/`) and +are stable across runs of the same scenario. + +```bash +agent-harness run scenarios/goal_hijack/basic.yaml \ + --trace-file traces/basic.json \ + --sarif-out results/basic.sarif +``` + +Upload to GitHub Code Scanning: + +```yaml +- name: Run security regression + run: | + agent-harness run scenarios/goal_hijack/basic.yaml \ + --trace-file traces/basic.json \ + --sarif-out results/basic.sarif + +- name: Upload SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: results/basic.sarif + if: always() +``` + +**Limitations:** SARIF output reflects regression assertion results, not +traditional code scanning findings. There are no source locations because +assertions evaluate runtime agent behaviour rather than static code. ## How pass and fail actually work `agent-harness run` writes machine-readable result JSON to the path you give From b3fb2efe970b16dfbc87ac019b8775661dc4d124 Mon Sep 17 00:00:00 2001 From: omobolaji adeyan Date: Thu, 18 Jun 2026 13:52:46 -0500 Subject: [PATCH 5/5] fix: sort SARIF CLI import --- src/agent_harness/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent_harness/cli.py b/src/agent_harness/cli.py index 95fa12b..70c5964 100644 --- a/src/agent_harness/cli.py +++ b/src/agent_harness/cli.py @@ -10,7 +10,6 @@ from agent_harness.adapters import DEFAULT_HTTP_TIMEOUT_SECONDS, AdapterError from agent_harness.junit import result_to_junit_xml -from agent_harness.sarif import result_to_sarif from agent_harness.runner import ( dry_run_scenario, run_scenario_live, @@ -21,6 +20,7 @@ run_scenario_with_python_target, run_scenario_with_trace, ) +from agent_harness.sarif import result_to_sarif from agent_harness.scenario import ScenarioValidationError, load_scenario from agent_harness.trace import TraceValidationError, load_trace