Switch Python parser to AST

Author: xnuinside

ARCHITECTURE.md

@@ -47,10 +47,8 @@ This allows adding new languages without changing core graph orchestration.
 
 #### Python Parser (`codegraph/parsers/python_parser.py`)
 
-Uses the legacy token-based parser in `codegraph/parser.py` to extract classes,
-functions, imports, and line ranges.
-
-The parser uses Python's `tokenize` module to extract code structure from source files.
+Uses Python's `ast` (and `typed_ast` for Python 2.x) to extract classes, functions,
+imports, and line ranges.
 
 #### Rust Parser (`codegraph/parsers/rust_parser.py`)
 

codegraph/core.py

@@ -7,7 +7,7 @@
 class CodeGraph:
     def __init__(self, args: Namespace):
         language = getattr(args, "language", "python")
-        self.parser = get_parser(language)
+        self.parser = get_parser(language, args=args)
         self.paths_list = self.parser.get_source_files(args.paths)
         self.modules_data = self.parser.parse_files(self.paths_list)
 
@@ -22,10 +22,11 @@ def get_lines_numbers(self):
                 first number in tuple - start line, second - last line
         """
         data = {}
-        for module in self.modules_data:
+        metadata = self.get_entity_metadata()
+        for module, entities in metadata.items():
             data[module] = {}
-            for func in self.modules_data[module]:
-                data[module][func.name] = (func.lineno, func.endno)
+            for name, info in entities.items():
+                data[module][name] = (info.get("lineno"), info.get("endno"))
         return data
 
     def get_entity_metadata(self) -> Dict:

codegraph/main.py

@@ -48,7 +48,11 @@
     show_default=True,
     help="Language parser to use",
 )
-def cli(paths, object_only, file_path, distance, matplotlib, output, csv, language):
+@click.option(
+    "--python-version",
+    help="Target Python version for parsing (e.g. 2.7, 3.8, 3.10)",
+)
+def cli(paths, object_only, file_path, distance, matplotlib, output, csv, language, python_version):
     """
     Tool that creates a graph of code to show dependencies between code entities (methods, classes, etc.).
     CodeGraph does not execute code, it is based only on lex and syntax parsing.
@@ -71,6 +75,7 @@ def cli(paths, object_only, file_path, distance, matplotlib, output, csv, langua
         output=output,
         csv=csv,
         language=language,
+        python_version=python_version,
     )
     main(args)