Skip to content

Commit df61807

Browse files
committed
fix(safety): normalize remaining CRLF files to LF (.gitignore, tool_safety_eval.py)
1 parent 70501b2 commit df61807

2 files changed

Lines changed: 130 additions & 131 deletions

File tree

.gitignore

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,26 @@
1-
.idea
2-
.vscode
3-
.DS_Store
4-
*.lock
5-
*.log
6-
examples/*.log
7-
8-
trpc-agent-py.egg-info
9-
10-
build
11-
dist
12-
.venv
13-
venv
14-
venv*
15-
__pycache__
16-
17-
.coverage
18-
htmlcov
19-
cov.tmp
20-
coverage.xml
21-
test-ngtest-ut-trpc-agent-py.xml
22-
.pytest_cache
23-
24-
node_modules
25-
package-lock.json
26-
pyrightconfig.json
27-
1+
.idea
2+
.vscode
3+
.DS_Store
4+
*.lock
5+
*.log
6+
examples/*.log
7+
8+
trpc-agent-py.egg-info
9+
10+
build
11+
dist
12+
.venv
13+
venv
14+
venv*
15+
__pycache__
16+
17+
.coverage
18+
htmlcov
19+
cov.tmp
20+
coverage.xml
21+
test-ngtest-ut-trpc-agent-py.xml
22+
.pytest_cache
23+
24+
node_modules
25+
package-lock.json
26+
pyrightconfig.json

scripts/tool_safety_eval.py

Lines changed: 104 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -1,104 +1,104 @@
1-
#!/usr/bin/env python3
2-
# Tencent is pleased to support the open source community by making tRPC-Agent-Python available.
3-
#
4-
# Copyright (C) 2026 Tencent. All rights reserved.
5-
#
6-
# tRPC-Agent-Python is licensed under Apache-2.0.
7-
"""Evaluate Tool Script Safety Guard detection / false-positive rates.
8-
9-
Reads a manifest.yaml of the form::
10-
11-
cases:
12-
- file: 02_dangerous_delete.sh
13-
expect: deny
14-
risk: high
15-
- file: 01_safe_python.py
16-
expect: allow
17-
18-
Prints detection rate, false-positive rate, and average scan duration.
19-
"""
20-
from __future__ import annotations
21-
22-
import argparse
23-
import sys
24-
import time
25-
from pathlib import Path
26-
27-
28-
def main(argv=None) -> int:
29-
repo_root = Path(__file__).resolve().parents[1]
30-
if str(repo_root) not in sys.path:
31-
sys.path.insert(0, str(repo_root))
32-
33-
parser = argparse.ArgumentParser(description="Evaluate tool safety detection rates")
34-
parser.add_argument("--samples", required=True, help="Samples directory")
35-
parser.add_argument("--manifest", required=True, help="manifest.yaml path")
36-
parser.add_argument(
37-
"--policy",
38-
default="examples/tool_safety/tool_safety_policy.yaml",
39-
help="Policy YAML path",
40-
)
41-
args = parser.parse_args(argv)
42-
43-
import yaml
44-
from trpc_agent_sdk.safety import PolicyConfig
45-
from trpc_agent_sdk.safety import SafetyScanner
46-
from trpc_agent_sdk.safety import ScanInput
47-
48-
policy = PolicyConfig.from_yaml(args.policy)
49-
scanner = SafetyScanner(policy=policy)
50-
samples_dir = Path(args.samples)
51-
manifest = yaml.safe_load(Path(args.manifest).read_text(encoding="utf-8")) or {}
52-
cases = manifest.get("cases") or manifest.get("samples") or []
53-
54-
dangerous_total = 0
55-
dangerous_hit = 0
56-
safe_total = 0
57-
safe_fp = 0
58-
durations = []
59-
60-
for case in cases:
61-
name = case.get("file") or case.get("name")
62-
expect = case.get("expect") or case.get("decision")
63-
path = samples_dir / name
64-
if not path.is_file():
65-
print(f"missing sample: {name}", file=sys.stderr)
66-
continue
67-
script = path.read_text(encoding="utf-8")
68-
lang = "python" if path.suffix == ".py" else "bash"
69-
t0 = time.perf_counter()
70-
report = scanner.scan(ScanInput(script=script, language=lang, tool_name=name))
71-
durations.append((time.perf_counter() - t0) * 1000)
72-
73-
if expect == "allow":
74-
safe_total += 1
75-
if report.decision.value != "allow":
76-
safe_fp += 1
77-
print(f"FP {name}: got {report.decision.value}")
78-
else:
79-
dangerous_total += 1
80-
# For needs_human_review expectations, accept exact match;
81-
# for deny, require deny.
82-
if expect == "needs_human_review":
83-
if report.decision.value in ("needs_human_review", "deny"):
84-
dangerous_hit += 1
85-
else:
86-
print(f"MISS {name}: got {report.decision.value}, expect {expect}")
87-
elif report.decision.value == expect:
88-
dangerous_hit += 1
89-
else:
90-
print(f"MISS {name}: got {report.decision.value}, expect {expect}")
91-
92-
det = (dangerous_hit / dangerous_total) if dangerous_total else 1.0
93-
fpr = (safe_fp / safe_total) if safe_total else 0.0
94-
avg_ms = sum(durations) / len(durations) if durations else 0.0
95-
print(f"detection_rate={det:.1%} ({dangerous_hit}/{dangerous_total}) "
96-
f"false_positive_rate={fpr:.1%} ({safe_fp}/{safe_total}) "
97-
f"avg_scan_ms={avg_ms:.3f}")
98-
if det < 0.9 or fpr > 0.1:
99-
return 1
100-
return 0
101-
102-
103-
if __name__ == "__main__":
104-
sys.exit(main())
1+
#!/usr/bin/env python3
2+
# Tencent is pleased to support the open source community by making tRPC-Agent-Python available.
3+
#
4+
# Copyright (C) 2026 Tencent. All rights reserved.
5+
#
6+
# tRPC-Agent-Python is licensed under Apache-2.0.
7+
"""Evaluate Tool Script Safety Guard detection / false-positive rates.
8+
9+
Reads a manifest.yaml of the form::
10+
11+
cases:
12+
- file: 02_dangerous_delete.sh
13+
expect: deny
14+
risk: high
15+
- file: 01_safe_python.py
16+
expect: allow
17+
18+
Prints detection rate, false-positive rate, and average scan duration.
19+
"""
20+
from __future__ import annotations
21+
22+
import argparse
23+
import sys
24+
import time
25+
from pathlib import Path
26+
27+
28+
def main(argv=None) -> int:
29+
repo_root = Path(__file__).resolve().parents[1]
30+
if str(repo_root) not in sys.path:
31+
sys.path.insert(0, str(repo_root))
32+
33+
parser = argparse.ArgumentParser(description="Evaluate tool safety detection rates")
34+
parser.add_argument("--samples", required=True, help="Samples directory")
35+
parser.add_argument("--manifest", required=True, help="manifest.yaml path")
36+
parser.add_argument(
37+
"--policy",
38+
default="examples/tool_safety/tool_safety_policy.yaml",
39+
help="Policy YAML path",
40+
)
41+
args = parser.parse_args(argv)
42+
43+
import yaml
44+
from trpc_agent_sdk.safety import PolicyConfig
45+
from trpc_agent_sdk.safety import SafetyScanner
46+
from trpc_agent_sdk.safety import ScanInput
47+
48+
policy = PolicyConfig.from_yaml(args.policy)
49+
scanner = SafetyScanner(policy=policy)
50+
samples_dir = Path(args.samples)
51+
manifest = yaml.safe_load(Path(args.manifest).read_text(encoding="utf-8")) or {}
52+
cases = manifest.get("cases") or manifest.get("samples") or []
53+
54+
dangerous_total = 0
55+
dangerous_hit = 0
56+
safe_total = 0
57+
safe_fp = 0
58+
durations = []
59+
60+
for case in cases:
61+
name = case.get("file") or case.get("name")
62+
expect = case.get("expect") or case.get("decision")
63+
path = samples_dir / name
64+
if not path.is_file():
65+
print(f"missing sample: {name}", file=sys.stderr)
66+
continue
67+
script = path.read_text(encoding="utf-8")
68+
lang = "python" if path.suffix == ".py" else "bash"
69+
t0 = time.perf_counter()
70+
report = scanner.scan(ScanInput(script=script, language=lang, tool_name=name))
71+
durations.append((time.perf_counter() - t0) * 1000)
72+
73+
if expect == "allow":
74+
safe_total += 1
75+
if report.decision.value != "allow":
76+
safe_fp += 1
77+
print(f"FP {name}: got {report.decision.value}")
78+
else:
79+
dangerous_total += 1
80+
# For needs_human_review expectations, accept exact match;
81+
# for deny, require deny.
82+
if expect == "needs_human_review":
83+
if report.decision.value in ("needs_human_review", "deny"):
84+
dangerous_hit += 1
85+
else:
86+
print(f"MISS {name}: got {report.decision.value}, expect {expect}")
87+
elif report.decision.value == expect:
88+
dangerous_hit += 1
89+
else:
90+
print(f"MISS {name}: got {report.decision.value}, expect {expect}")
91+
92+
det = (dangerous_hit / dangerous_total) if dangerous_total else 1.0
93+
fpr = (safe_fp / safe_total) if safe_total else 0.0
94+
avg_ms = sum(durations) / len(durations) if durations else 0.0
95+
print(f"detection_rate={det:.1%} ({dangerous_hit}/{dangerous_total}) "
96+
f"false_positive_rate={fpr:.1%} ({safe_fp}/{safe_total}) "
97+
f"avg_scan_ms={avg_ms:.3f}")
98+
if det < 0.9 or fpr > 0.1:
99+
return 1
100+
return 0
101+
102+
103+
if __name__ == "__main__":
104+
sys.exit(main())

0 commit comments

Comments
 (0)