|
| 1 | +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. |
| 2 | +# |
| 3 | +# Copyright (C) 2026 Tencent. All rights reserved. |
| 4 | +# |
| 5 | +# tRPC-Agent-Python is licensed under Apache-2.0. |
| 6 | +"""Batch-scan the 12 sample scripts and report acceptance metrics. |
| 7 | +
|
| 8 | +Produces ``tool_safety_report.json`` and ``tool_safety_audit.jsonl`` next to this |
| 9 | +file, then prints the headline acceptance numbers: |
| 10 | +
|
| 11 | +- every sample scanned and reported, |
| 12 | +- high-risk (deny) detection rate, |
| 13 | +- safe-sample false-positive rate (safe -> deny only; review is not counted), |
| 14 | +- 100% detection of the three must-catch categories, |
| 15 | +- single 500-line scan duration. |
| 16 | +
|
| 17 | +Run:: |
| 18 | +
|
| 19 | + python examples/tool_safety_guard/run_scan.py |
| 20 | +""" |
| 21 | + |
| 22 | +from __future__ import annotations |
| 23 | + |
| 24 | +import json |
| 25 | +import time |
| 26 | +from pathlib import Path |
| 27 | + |
| 28 | +from trpc_agent_sdk.tools.safety import Decision |
| 29 | +from trpc_agent_sdk.tools.safety import Language |
| 30 | +from trpc_agent_sdk.tools.safety import AuditLogger |
| 31 | +from trpc_agent_sdk.tools.safety import SafetyEngine |
| 32 | +from trpc_agent_sdk.tools.safety import ScanInput |
| 33 | +from trpc_agent_sdk.tools.safety import load_policy |
| 34 | + |
| 35 | +HERE = Path(__file__).resolve().parent |
| 36 | +SAMPLES_DIR = HERE / "samples" |
| 37 | +POLICY_PATH = HERE / "tool_safety_policy.yaml" |
| 38 | +REPORT_PATH = HERE / "tool_safety_report.json" |
| 39 | +AUDIT_PATH = HERE / "tool_safety_audit.jsonl" |
| 40 | + |
| 41 | +# Samples in the must-catch categories (must be denied 100% of the time). |
| 42 | +MUST_CATCH = { |
| 43 | + "02_dangerous_delete.py": "dangerous_delete", |
| 44 | + "03_read_secret.py": "secret_read", |
| 45 | + "04_network_egress.py": "non_allowlisted_egress", |
| 46 | + "10_secret_leak_output.py": "secret_read", |
| 47 | + "11_curl_pipe_bash.sh": "non_allowlisted_egress", |
| 48 | +} |
| 49 | + |
| 50 | +_SUFFIX_LANG = {".py": Language.PYTHON, ".sh": Language.BASH, ".bash": Language.BASH} |
| 51 | + |
| 52 | + |
| 53 | +def detect_language(path: Path) -> Language: |
| 54 | + return _SUFFIX_LANG.get(path.suffix.lower(), Language.UNKNOWN) |
| 55 | + |
| 56 | + |
| 57 | +def main() -> int: |
| 58 | + expected = json.loads((SAMPLES_DIR / "EXPECTED.json").read_text(encoding="utf-8")) |
| 59 | + engine = SafetyEngine(load_policy(str(POLICY_PATH))) |
| 60 | + |
| 61 | + # Fresh audit log each run. |
| 62 | + if AUDIT_PATH.exists(): |
| 63 | + AUDIT_PATH.unlink() |
| 64 | + audit = AuditLogger(str(AUDIT_PATH)) |
| 65 | + |
| 66 | + reports: list[dict] = [] |
| 67 | + correct = 0 |
| 68 | + deny_total = deny_hit = 0 |
| 69 | + safe_total = safe_fp = 0 |
| 70 | + must_catch_hits: dict[str, bool] = {} |
| 71 | + |
| 72 | + for name in sorted(expected): |
| 73 | + path = SAMPLES_DIR / name |
| 74 | + script = path.read_text(encoding="utf-8", errors="ignore") |
| 75 | + report = engine.scan(ScanInput(script=script, tool_name=name, language=detect_language(path))) |
| 76 | + audit.log(report, blocked=report.decision == Decision.DENY) |
| 77 | + |
| 78 | + exp = expected[name] |
| 79 | + got = report.decision.value |
| 80 | + correct += int(got == exp) |
| 81 | + if exp == "deny": |
| 82 | + deny_total += 1 |
| 83 | + deny_hit += int(got == "deny") |
| 84 | + if exp == "allow": |
| 85 | + safe_total += 1 |
| 86 | + safe_fp += int(got == "deny") |
| 87 | + if name in MUST_CATCH: |
| 88 | + must_catch_hits[name] = (got == "deny") |
| 89 | + |
| 90 | + record = report.to_dict() |
| 91 | + record["file"] = name |
| 92 | + record["expected"] = exp |
| 93 | + reports.append(record) |
| 94 | + print(f" [{got:>18}] expected={exp:<18} {name}") |
| 95 | + |
| 96 | + # 500-line performance probe. |
| 97 | + big = "\n".join(f"value_{i} = {i} * {i}" for i in range(500)) |
| 98 | + start = time.perf_counter() |
| 99 | + engine.scan(ScanInput(script=big, tool_name="perf_probe", language=Language.PYTHON)) |
| 100 | + duration = time.perf_counter() - start |
| 101 | + |
| 102 | + summary = { |
| 103 | + "total": len(reports), |
| 104 | + "decision_match": f"{correct}/{len(reports)}", |
| 105 | + "deny_detection_rate": f"{deny_hit}/{deny_total}", |
| 106 | + "safe_false_positive_rate": f"{safe_fp}/{safe_total}", |
| 107 | + "must_catch_all_denied": all(must_catch_hits.values()), |
| 108 | + "scan_500_lines_seconds": round(duration, 4), |
| 109 | + } |
| 110 | + REPORT_PATH.write_text( |
| 111 | + json.dumps({"summary": summary, "reports": reports}, ensure_ascii=False, indent=2), |
| 112 | + encoding="utf-8") |
| 113 | + |
| 114 | + print("\n=== acceptance metrics ===") |
| 115 | + print(f"all samples scanned & reported : {len(reports)}/12") |
| 116 | + print(f"decision match : {summary['decision_match']}") |
| 117 | + print(f"high-risk (deny) detection : {summary['deny_detection_rate']}") |
| 118 | + print(f"safe false-positive (safe->deny): {summary['safe_false_positive_rate']}") |
| 119 | + print(f"must-catch categories 100% : {summary['must_catch_all_denied']} {must_catch_hits}") |
| 120 | + print(f"500-line scan duration : {summary['scan_500_lines_seconds']}s") |
| 121 | + print(f"\nreport: {REPORT_PATH}\naudit : {AUDIT_PATH}") |
| 122 | + return 0 |
| 123 | + |
| 124 | + |
| 125 | +if __name__ == "__main__": |
| 126 | + raise SystemExit(main()) |
0 commit comments