Fix PythonPlugin builtin module discovery

Author: barnabasdomozi

plugins/python/parser/pyparser/parser.py

@@ -30,6 +30,8 @@ def parseProject(settings, n_proc):
         ast_function_signature=settings["ast_function_signature"],
         file_refs=settings["file_refs"]
     )
+    config.debug = True
+    config.stack_trace = True
 
     log(f"Parsing project: {config.root_path}")
     log_config(config)

plugins/python/parser/pyparser/pybuiltin.py

@@ -2,34 +2,44 @@
 import os
 import importlib.util
 import traceback
+import sysconfig
 from jedi.api.classes import Name
 from parserlog import log, bcolors
 from parserconfig import ParserConfig
 
 class PYBuiltin:
     builtin = {}
 
+    @staticmethod
+    def __searchDirectory(directory: str):
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                p = os.path.join(root, file)
+                ext = os.path.splitext(p)[1]
+
+                if ext and ext.lower() == '.py':
+                    PYBuiltin.builtin[p] = True
+
     @staticmethod
     def findBuiltins(config: ParserConfig):
         try:
+            # Consider all Python modules in the stdlib directory builtin
+            sysconfig_paths = sysconfig.get_paths()
+            if "stdlib" in sysconfig_paths:
+                PYBuiltin.__searchDirectory(sysconfig_paths["stdlib"])
+
+            # Locate module paths via ModuleSpec
+            # However, this can return "frozen" and "built-in" as module origin and not the actual file
             # Note: Python 3.10+ required
             stdlib_modules = sys.stdlib_module_names
-
             for e in stdlib_modules:
                 spec = importlib.util.find_spec(e)
-                if spec and spec.origin:
+                if spec and spec.origin and spec.origin != "frozen" and spec.origin != "built-in":
                     PYBuiltin.builtin[spec.origin] = True
 
                 if spec and spec.submodule_search_locations:
                     for submodule_dir in spec.submodule_search_locations:
-                        for root, dirs, files in os.walk(submodule_dir):
-                            for file in files:
-                                p = os.path.join(root, file)
-                                ext = os.path.splitext(p)[1]
-
-                                if ext and ext.lower() == '.py':
-                                    PYBuiltin.builtin[p] = True
-
+                        PYBuiltin.__searchDirectory(submodule_dir)
         except:
             log(f"{bcolors.FAIL}Failed to find Python builtins!")
             if config.stack_trace:

plugins/python/parser/pyparser/pyname.py

@@ -98,6 +98,7 @@ def getNodeInfo(self) -> NodeInfo:
         node.is_definition = self.name.is_definition()
         node.file_id = self.__getFileId()
         node.type_hint = self.__getNameTypeHint()
+        node.type_hint = str(self.path) + ":" + str(list(map(lambda x : x.path, self.defs)))
         node.is_builtin = PYBuiltin.isBuiltin(self.name) or any(list(map(lambda x : PYBuiltin.isBuiltin(x.name), self.defs)))
 
         parent = self.name.parent()

plugins/python/test/CMakeLists.txt

@@ -54,6 +54,7 @@ add_test(NAME pythonparser COMMAND pythonparsertest
        --name pythonparsertest \
        --input ${CMAKE_CURRENT_SOURCE_DIR}/sources/ \
        --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \
+       --venvpath ${INSTALL_PYTHON_DIR}/venv \
        --force"
     "${TEST_DB}")
 
@@ -64,5 +65,6 @@ add_test(NAME pythonservice COMMAND pythonservicetest
        --name pythonservicetest \
        --input ${CMAKE_CURRENT_SOURCE_DIR}/sources/ \
        --workspace ${CMAKE_CURRENT_BINARY_DIR}/workdir/ \
+       --venvpath ${INSTALL_PYTHON_DIR}/venv \
        --force"
     "${TEST_DB}")

plugins/python/test/src/pythonparsertest.cpp

@@ -26,15 +26,14 @@ class PythonParserTest : public ::testing::Test
 
   model::PYName queryFile(const std::string& filename, const odb::query<model::PYName>& odb_query)
   {
-    model::PYName pyname;
-    if (m_files.count(filename))
+    if(!m_files.count(filename))
     {
-      _transaction([&, this]() {
-        pyname = _db->query_value<model::PYName>(odb_query && odb::query<model::PYName>::file_id == m_files[filename]);
-      });
+      throw std::runtime_error("File " + filename + " not found!");
     }
 
-    return pyname;
+    return _transaction([&, this]() {
+      return _db->query_value<model::PYName>(odb_query && odb::query<model::PYName>::file_id == m_files[filename]);
+    });
   }
 
 private:
@@ -422,6 +421,8 @@ TEST_F(PythonParserTest, BuiltinVariable)
         (odb::query<model::PYName>::line_start == 6 &&
         odb::query<model::PYName>::value == "print"));
 
+  LOG(info) << pyname.type_hint;
+
   EXPECT_EQ(pyname.is_builtin, true);
 
   pyname = queryFile("imports.py",