Skip to content

Commit f1c5a82

Browse files
committed
feat(reachability): extend opt-in library-mode to all parsers + CLI
#117's library-mode (seed the exported public API so a library with no main/ route/CLI entry point isn't blacked out) lived only in the Python path (core/parser_adapter). The other six parsers run as subprocesses with their own reachability copy, so a C/JS/etc. library still collapsed: tree-sitter's C core pruned 661 -> 24 (all wasm), the public ts_parser_* API never seeded. Centralize _library_seed_ids into utilities/agentic_enhancer.library_seed_ids (now handles both is_exported snake_case on disk and isExported camelCase from the pipelines' in-memory normalize). Thread an opt-in library_mode through all 4 parallel surfaces for each of the 6 subprocess parsers (parse dispatch -> _parse_<lang> subprocess cmd -> test_pipeline --library-mode argparse -> union library_seed_ids into entry_points before the BFS), plus scan_repository and the 'openant parse' / 'openant scan' CLI flags. Union-only: never drops a structurally detected entry point, so app scans are unaffected. Verified end-to-end on tree-sitter C: without the flag the blackout warning fires (24/661); with --library-mode, 352 public-API seeds -> 550 reachable, the parser core (parser.c/lexer.c/stack.c/subtree.c/query.c/node.c) now analysable. Tests: 8 new (library_seed_ids both casings + name heuristic); #117 Python path unchanged (6 green).
1 parent b003346 commit f1c5a82

12 files changed

Lines changed: 232 additions & 57 deletions

File tree

libs/openant-core/core/parser_adapter.py

Lines changed: 30 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -114,17 +114,17 @@ def parse_repository(
114114
if language == "python":
115115
result = _parse_python(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
116116
elif language == "javascript":
117-
result = _parse_javascript(repo_path, output_dir, processing_level, skip_tests, name)
117+
result = _parse_javascript(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
118118
elif language == "go":
119-
result = _parse_go(repo_path, output_dir, processing_level, skip_tests, name)
119+
result = _parse_go(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
120120
elif language == "c":
121-
result = _parse_c(repo_path, output_dir, processing_level, skip_tests, name)
121+
result = _parse_c(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
122122
elif language == "ruby":
123-
result = _parse_ruby(repo_path, output_dir, processing_level, skip_tests, name)
123+
result = _parse_ruby(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
124124
elif language == "php":
125-
result = _parse_php(repo_path, output_dir, processing_level, skip_tests, name)
125+
result = _parse_php(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
126126
elif language == "zig":
127-
result = _parse_zig(repo_path, output_dir, processing_level, skip_tests, name)
127+
result = _parse_zig(repo_path, output_dir, processing_level, skip_tests, name, library_mode)
128128
else:
129129
raise ValueError(f"Unsupported language: {language}")
130130

@@ -192,26 +192,10 @@ def _maybe_apply_diff_filter(
192192
# Reachability filter (shared by Python path; JS/Go handle it internally)
193193
# ---------------------------------------------------------------------------
194194

195-
def _library_seed_ids(functions: dict) -> "set[str]":
196-
"""Public-API seed set for library-mode reachability.
197-
198-
A pure library exposes no main/route/CLI entry point, so the structural
199-
detector finds nothing and the whole library is filtered out (0 reachable).
200-
In library-mode the *public surface* IS the entry surface: seed every
201-
exported/public function and let the forward BFS pull in its callees.
202-
203-
Public = exported AND not name-private. ``is_exported`` is honoured when the
204-
parser provides it (C/Go/JS — excludes ``static``/unexported); for parsers
205-
without the field (python/ruby/php) it defaults True and the name heuristic
206-
(leading underscore = private) decides. The bias is intentionally toward
207-
over-seeding (more reachable = more analysed), never under-seeding.
208-
"""
209-
seeds: set[str] = set()
210-
for func_id, fd in functions.items():
211-
name = (fd.get("name") or func_id.rsplit(":", 1)[-1]).split(".")[-1]
212-
if fd.get("is_exported", True) and not name.startswith("_"):
213-
seeds.add(func_id)
214-
return seeds
195+
# library_seed_ids is now shared in utilities/agentic_enhancer/entry_point_detector.py
196+
# so every parser pipeline (not just Python) can seed the public API. It is loaded
197+
# below via the same importlib path as EntryPointDetector to dodge the heavy
198+
# utilities/__init__ imports.
215199

216200

217201
def apply_reachability_filter(
@@ -263,6 +247,7 @@ def _load_module(name, filename):
263247
_ra = _load_module("reachability_analyzer", "reachability_analyzer.py")
264248
EntryPointDetector = _epd.EntryPointDetector
265249
blackout_warning = _epd.blackout_warning
250+
library_seed_ids = _epd.library_seed_ids
266251
ReachabilityAnalyzer = _ra.ReachabilityAnalyzer
267252

268253
call_graph_path = os.path.join(output_dir, "call_graph.json")
@@ -290,7 +275,7 @@ def _load_module(name, filename):
290275
# never demotes a structurally-detected app entry point, so an app scan with
291276
# the flag on can only gain reachable units, never lose one.
292277
if library_mode:
293-
entry_points = entry_points | _library_seed_ids(functions)
278+
entry_points = entry_points | library_seed_ids(functions)
294279

295280
units = dataset.get("units", [])
296281
original_count = len(units)
@@ -544,7 +529,7 @@ def _file_lock(lock_path: Path):
544529
f.close()
545530

546531

547-
def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult:
532+
def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult:
548533
"""Invoke the JavaScript/TypeScript parser.
549534
550535
The JS parser is a PipelineTest class that runs Node.js subprocesses.
@@ -568,6 +553,8 @@ def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, sk
568553
cmd.extend(["--name", name])
569554
if skip_tests:
570555
cmd.append("--skip-tests")
556+
if library_mode:
557+
cmd.append("--library-mode")
571558

572559
result = subprocess.run(
573560
cmd,
@@ -603,7 +590,7 @@ def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, sk
603590
# Go parser
604591
# ---------------------------------------------------------------------------
605592

606-
def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult:
593+
def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult:
607594
"""Invoke the Go parser.
608595
609596
The Go parser is a PipelineTest class that calls a compiled Go binary.
@@ -624,6 +611,8 @@ def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests
624611
cmd.extend(["--name", name])
625612
if skip_tests:
626613
cmd.append("--skip-tests")
614+
if library_mode:
615+
cmd.append("--library-mode")
627616

628617
result = subprocess.run(
629618
cmd,
@@ -659,7 +648,7 @@ def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests
659648
# C/C++ parser
660649
# ---------------------------------------------------------------------------
661650

662-
def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult:
651+
def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult:
663652
"""Invoke the C/C++ parser.
664653
665654
The C parser uses tree-sitter for function extraction and call graph
@@ -682,6 +671,8 @@ def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests:
682671
cmd.extend(["--name", name])
683672
if skip_tests:
684673
cmd.append("--skip-tests")
674+
if library_mode:
675+
cmd.append("--library-mode")
685676

686677
result = subprocess.run(
687678
cmd,
@@ -718,7 +709,7 @@ def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests:
718709
# Ruby parser
719710
# ---------------------------------------------------------------------------
720711

721-
def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult:
712+
def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult:
722713
"""Invoke the Ruby parser.
723714
724715
The Ruby parser uses tree-sitter for function extraction and call graph
@@ -741,6 +732,8 @@ def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tes
741732
cmd.extend(["--name", name])
742733
if skip_tests:
743734
cmd.append("--skip-tests")
735+
if library_mode:
736+
cmd.append("--library-mode")
744737

745738
result = subprocess.run(
746739
cmd,
@@ -777,7 +770,7 @@ def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tes
777770
# PHP parser
778771
# ---------------------------------------------------------------------------
779772

780-
def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult:
773+
def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult:
781774
"""Invoke the PHP parser.
782775
783776
The PHP parser uses tree-sitter for function extraction and call graph
@@ -800,6 +793,8 @@ def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_test
800793
cmd.extend(["--name", name])
801794
if skip_tests:
802795
cmd.append("--skip-tests")
796+
if library_mode:
797+
cmd.append("--library-mode")
803798

804799
result = subprocess.run(
805800
cmd,
@@ -836,7 +831,7 @@ def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_test
836831
# Zig parser
837832
# ---------------------------------------------------------------------------
838833

839-
def _parse_zig(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult:
834+
def _parse_zig(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult:
840835
"""Invoke the Zig parser.
841836
842837
The Zig parser uses tree-sitter for function extraction and call graph
@@ -859,6 +854,8 @@ def _parse_zig(repo_path: str, output_dir: str, processing_level: str, skip_test
859854
cmd.extend(["--name", name])
860855
if skip_tests:
861856
cmd.append("--skip-tests")
857+
if library_mode:
858+
cmd.append("--library-mode")
862859

863860
result = subprocess.run(
864861
cmd,

libs/openant-core/core/scanner.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ def scan_repository(
6262
diff_manifest: str | None = None,
6363
llm_reachability: bool = False,
6464
llm_reachability_max_code_bytes: int = 1500,
65+
library_mode: bool = False,
6566
) -> ScanResult:
6667
"""Scan a repository for vulnerabilities.
6768
@@ -154,6 +155,7 @@ def _step_label(name: str) -> str:
154155
processing_level=effective_parse_level,
155156
skip_tests=skip_tests,
156157
diff_manifest=diff_manifest,
158+
library_mode=library_mode,
157159
)
158160

159161
ctx.summary = {

libs/openant-core/openant/cli.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ def cmd_scan(args):
7575
repo_url=getattr(args, "repo_url", None),
7676
commit_sha=getattr(args, "commit_sha", None),
7777
diff_manifest=getattr(args, "diff_manifest", None),
78+
library_mode=getattr(args, "library_mode", False),
7879
llm_reachability=getattr(args, "llm_reachability", False),
7980
llm_reachability_max_code_bytes=getattr(
8081
args, "llm_reachability_max_code_bytes", 1500
@@ -128,6 +129,7 @@ def cmd_parse(args):
128129
skip_tests=not args.no_skip_tests,
129130
name=getattr(args, "name", None),
130131
diff_manifest=getattr(args, "diff_manifest", None),
132+
library_mode=getattr(args, "library_mode", False),
131133
)
132134

133135
ctx.summary = {
@@ -983,6 +985,8 @@ def main():
983985
scan_p.add_argument("--dynamic-test", action="store_true",
984986
help="Enable Docker-isolated dynamic testing (off by default)")
985987
scan_p.add_argument("--no-skip-tests", action="store_true", help="Include test files in parsing (default: tests are skipped)")
988+
scan_p.add_argument("--library-mode", action="store_true",
989+
help="Seed the exported public API as entry points (for libraries with no main/route/CLI entry point)")
986990
scan_p.add_argument("--limit", type=int, help="Max units to analyze")
987991
scan_p.add_argument("--model", choices=["opus", "sonnet"], default="opus", help="Model (default: opus)")
988992
scan_p.add_argument("--workers", type=int, default=8,
@@ -1037,6 +1041,8 @@ def main():
10371041
help="Processing level (default: reachable)",
10381042
)
10391043
parse_p.add_argument("--no-skip-tests", action="store_true", help="Include test files in parsing (default: tests are skipped)")
1044+
parse_p.add_argument("--library-mode", action="store_true",
1045+
help="Seed the exported public API as entry points (for libraries with no main/route/CLI entry point)")
10401046
parse_p.add_argument("--name", help="Dataset name (default: derived from repo path)")
10411047
parse_p.add_argument("--diff-manifest", help="Path to diff_manifest.json; tags units with diff_selected")
10421048
parse_p.set_defaults(func=cmd_parse)

libs/openant-core/parsers/c/test_pipeline.py

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
4949
from utilities.file_io import open_utf8, read_json, run_utf8, write_json
5050
from utilities.context_enhancer import ContextEnhancer
51-
from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning
51+
from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids
5252

5353
# Local imports
5454
from repository_scanner import RepositoryScanner
@@ -87,7 +87,8 @@ def __init__(
8787
processing_level: ProcessingLevel = ProcessingLevel.ALL,
8888
skip_tests: bool = False,
8989
depth: int = 3,
90-
name: str = None
90+
name: str = None,
91+
library_mode: bool = False
9192
):
9293
self.repo_path = os.path.abspath(repo_path)
9394
self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output')
@@ -98,6 +99,7 @@ def __init__(
9899
self.skip_tests = skip_tests
99100
self.depth = depth
100101
self.dataset_name = name
102+
self.library_mode = library_mode
101103

102104
# Pipeline artifacts
103105
self.scan_results_file = None
@@ -292,6 +294,13 @@ def apply_reachability_filter(self) -> bool:
292294
detector = EntryPointDetector(normalized_functions, call_graph)
293295
self.entry_points = detector.detect_entry_points()
294296

297+
# Library-mode: a library's entry surface is its exported public API,
298+
# which carries no main/route/CLI marker. Seed it so the BFS reaches
299+
# the core instead of blacking out. Union-only — never drops a
300+
# structurally-detected entry point.
301+
if self.library_mode:
302+
self.entry_points = self.entry_points | library_seed_ids(normalized_functions)
303+
295304
# Build reachability
296305
reachability = ReachabilityAnalyzer(
297306
functions=normalized_functions,
@@ -1020,6 +1029,11 @@ def main():
10201029
default=None,
10211030
help='Dataset name (default: derived from repo path)'
10221031
)
1032+
parser.add_argument(
1033+
'--library-mode',
1034+
action='store_true',
1035+
help='Seed the exported public API as entry points (for libraries with no main/route/CLI)'
1036+
)
10231037

10241038
args = parser.parse_args()
10251039

@@ -1041,7 +1055,8 @@ def main():
10411055
processing_level=processing_level,
10421056
skip_tests=args.skip_tests,
10431057
depth=args.depth,
1044-
name=args.name
1058+
name=args.name,
1059+
library_mode=args.library_mode
10451060
)
10461061
results = pipeline.run_full_pipeline()
10471062

libs/openant-core/parsers/go/test_pipeline.py

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ def _stdout_supports_unicode() -> bool:
121121
# Add parent directory to path for utilities import
122122
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
123123
from utilities.context_enhancer import ContextEnhancer
124-
from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning
124+
from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids
125125

126126

127127
class ProcessingLevel(Enum):
@@ -152,7 +152,8 @@ def __init__(
152152
processing_level: ProcessingLevel = ProcessingLevel.ALL,
153153
skip_tests: bool = False,
154154
depth: int = 3,
155-
name: str = None
155+
name: str = None,
156+
library_mode: bool = False
156157
):
157158
self.repo_path = os.path.abspath(repo_path)
158159
self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output')
@@ -163,6 +164,7 @@ def __init__(
163164
self.skip_tests = skip_tests
164165
self.depth = depth
165166
self.dataset_name = name
167+
self.library_mode = library_mode
166168

167169
# Go parser binary location
168170
self.go_parser = os.path.join(self.parser_dir, 'go_parser', 'go_parser')
@@ -433,6 +435,11 @@ def apply_reachability_filter(self) -> bool:
433435
detector = EntryPointDetector(normalized_functions, call_graph)
434436
self.entry_points = detector.detect_entry_points()
435437

438+
# Library-mode: seed the exported public API (a library has no
439+
# main/route/CLI marker). Union-only — never drops a real entry point.
440+
if self.library_mode:
441+
self.entry_points = self.entry_points | library_seed_ids(normalized_functions)
442+
436443
# Build reachability analyzer
437444
reachability = ReachabilityAnalyzer(
438445
functions=normalized_functions,
@@ -1198,6 +1205,11 @@ def main():
11981205
default=None,
11991206
help='Dataset name (default: derived from repo path)'
12001207
)
1208+
parser.add_argument(
1209+
'--library-mode',
1210+
action='store_true',
1211+
help='Seed the exported public API as entry points (for libraries with no main/route/CLI)'
1212+
)
12011213

12021214
args = parser.parse_args()
12031215

@@ -1220,7 +1232,8 @@ def main():
12201232
processing_level=processing_level,
12211233
skip_tests=args.skip_tests,
12221234
depth=args.depth,
1223-
name=args.name
1235+
name=args.name,
1236+
library_mode=args.library_mode
12241237
)
12251238
results = pipeline.run_full_pipeline()
12261239

0 commit comments

Comments
 (0)