SONARPY-3212 Create custom stubs for boto3.client (#412)

maksim-grebeniuk-sonarsource · thomas-serre-sonarsource · sonartech · commit 2e7d9588975a · 2025-07-30T15:09:02.000Z
Co-authored-by: Thomas Serre &lt;thomas.serre@sonarsource.com&gt;
GitOrigin-RevId: fb83bdf542484b67e9ce1ef23834797cebc83e9b
diff --git a/python-frontend/src/main/resources/org/sonar/python/types/custom_protobuf/boto3.protobuf b/python-frontend/src/main/resources/org/sonar/python/types/custom_protobuf/boto3.protobuf
@@ -0,0 +1,15 @@
+
+boto3x
+clientboto3.client"8
+botocore.client.BaseClient"botocore.client.BaseClient*
+args
+Any*
+kwargs
+Any*f
+__path__boto3.__path__J
+builtins.list[builtins.str]
+builtins.str"builtins.str"builtins.list*�
+__annotations__boto3.__annotations__W
+builtins.dict[builtins.str,Any]
+builtins.str"builtins.str
+Any"builtins.dict
diff --git a/python-frontend/src/main/resources/org/sonar/python/types/custom_protobuf/botocore.client.protobuf b/python-frontend/src/main/resources/org/sonar/python/types/custom_protobuf/botocore.client.protobuf
@@ -0,0 +1,18 @@
+
+botocore.client�
+
+BaseClientbotocore.client.BaseClient"*SonarPythonAnalyzerFakeStub.CustomStubBase*�
+invoke!botocore.client.BaseClient.invoke"
+Any*B
+self8
+botocore.client.BaseClient"botocore.client.BaseClient*.
+FunctionName
+builtins.str"builtins.str*0
+InvocationType
+builtins.str"builtins.str*)
+Payload
+builtins.str"builtins.str*�
+__annotations__botocore.client.__annotations__W
+builtins.dict[builtins.str,Any]
+builtins.str"builtins.str
+Any"builtins.dict
diff --git a/python-frontend/src/main/resources/org/sonar/python/types/custom_protobuf/botocore.protobuf b/python-frontend/src/main/resources/org/sonar/python/types/custom_protobuf/botocore.protobuf
@@ -0,0 +1,9 @@
+
+botocore*i
+__path__botocore.__path__J
+builtins.list[builtins.str]
+builtins.str"builtins.str"builtins.list*�
+__annotations__botocore.__annotations__W
+builtins.dict[builtins.str,Any]
+builtins.str"builtins.str
+Any"builtins.dict
diff --git a/python-frontend/src/main/resources/org/sonar/python/types/third_party_protobuf_mypy/importlib_metadata._collections.protobuf b/python-frontend/src/main/resources/org/sonar/python/types/third_party_protobuf_mypy/importlib_metadata._collections.protobuf
diff --git a/python-frontend/src/main/resources/org/sonar/python/types/third_party_protobuf_mypy/importlib_metadata._typing.protobuf b/python-frontend/src/main/resources/org/sonar/python/types/third_party_protobuf_mypy/importlib_metadata._typing.protobuf
@@ -0,0 +1,10 @@
+
+importlib_metadata._typing*�
+__annotations__*importlib_metadata._typing.__annotations__W
+builtins.dict[builtins.str,Any]
+builtins.str"builtins.str
+Any"builtins.dict*�
+md_none"importlib_metadata._typing.md_noneu
+functools.partial[typing._T]B
+	typing._T"
+builtins.object"builtins.object"builtins.object"functools.partial
diff --git a/python-frontend/src/main/resources/org/sonar/python/types/third_party_protobuf_mypy/importlib_metadata.protobuf b/python-frontend/src/main/resources/org/sonar/python/types/third_party_protobuf_mypy/importlib_metadata.protobuf
diff --git a/python-frontend/typeshed_serializer/build-with-docker.sh b/python-frontend/typeshed_serializer/build-with-docker.sh
@@ -2,4 +2,5 @@
 
 export USER_ID="$(id -u)"
 export GROUP_ID="$(id -g)"
-docker compose -f ./docker/docker-compose.yml --project-directory ./docker up
+docker compose -f ./docker/docker-compose.yml --project-directory ./docker down
+docker compose -f ./docker/docker-compose.yml --project-directory ./docker up
diff --git a/python-frontend/typeshed_serializer/checksum b/python-frontend/typeshed_serializer/checksum
@@ -1,2 +1,2 @@
-77a91b4649947b77b314140e6c29c78f69cbe4303e4e007ac97938a7d4bbef31
-4a3074a882ea1c3b01720906430df1bf100c2edf10ba8b5fbc750e7961a404ea
+89637a19e32c219552eca326cce354337caa0b2033e092712dbb81fffbb5ad4b
+23cc6fb7a4cce09f3b84d2fbd0cdc0f2eb5b910f035bda950d5f840e80335468
diff --git a/python-frontend/typeshed_serializer/docker/Dockerfile b/python-frontend/typeshed_serializer/docker/Dockerfile
@@ -1,5 +1,13 @@
 FROM python:3.9
 
+RUN apt-get update && \
+    apt-get install -y protobuf-compiler && \
+    rm -rf /var/lib/apt/lists/*
+
+# Verify that protoc is installed and accessible on the PATH
+# This command will print the version of protoc, confirming its installation
+RUN protoc --version
+
 # Install tox
 RUN pip install --no-cache-dir tox
 
diff --git a/python-frontend/typeshed_serializer/docker/cmd.sh b/python-frontend/typeshed_serializer/docker/cmd.sh
@@ -1,6 +1,12 @@
 #!/bin/sh
 
+mkdir -p /sonar-python/python-frontend/typeshed_serializer/serializer/proto_out
+cd /sonar-python/python-frontend/src/main/protobuf
+
+# SONARPY-3216 - to investigate how to deal with it properly
+protoc -I=. --python_out=../../../typeshed_serializer/serializer/proto_out ./symbols.proto
+
 cd /sonar-python/python-frontend/typeshed_serializer
 # Recreate the env to make sure that latest dependencies will be downloaded
 python -m tox --recreate --notest
-python runners/tox_runner.py --skip_tests=true
+python runners/tox_runner.py --skip_tests=false
diff --git a/python-frontend/typeshed_serializer/requirements.txt b/python-frontend/typeshed_serializer/requirements.txt
@@ -1,6 +1,6 @@
 mypy==1.3.0
 mypy-extensions==1.0.0
-protobuf==5.29.0
+protobuf==5.29.3
 six==1.16.0
 toml==0.10.2
 typing-extensions==3.10.0.0
diff --git a/python-frontend/typeshed_serializer/resources/custom/boto3/__init__.pyi b/python-frontend/typeshed_serializer/resources/custom/boto3/__init__.pyi
@@ -0,0 +1,3 @@
+from botocore.client import BaseClient
+
+def client(*args, **kwargs) -> BaseClient: ...
diff --git a/python-frontend/typeshed_serializer/resources/custom/botocore/__init__.pyi b/python-frontend/typeshed_serializer/resources/custom/botocore/__init__.pyi
diff --git a/python-frontend/typeshed_serializer/resources/custom/botocore/client.pyi b/python-frontend/typeshed_serializer/resources/custom/botocore/client.pyi
@@ -0,0 +1,5 @@
+from SonarPythonAnalyzerFakeStub import CustomStubBase
+from typing import Any
+
+class BaseClient(CustomStubBase):
+    def invoke(self, FunctionName: str, InvocationType: str, Payload: str) -> Any: ...
diff --git a/python-frontend/typeshed_serializer/resources/importer/sonar_third_party_libs.py b/python-frontend/typeshed_serializer/resources/importer/sonar_third_party_libs.py
@@ -59,3 +59,4 @@
 import xxhash
 import yaml
 import fastapi
+import boto3
diff --git a/python-frontend/typeshed_serializer/runners/tox_runner.py b/python-frontend/typeshed_serializer/runners/tox_runner.py
@@ -157,6 +157,8 @@ def main(skip_tests=False, fail_fast=False):
     parser.add_argument('--skip_tests')
     parser.add_argument('--fail_fast')
     args = parser.parse_args()
+    logger.info(f'skip_tests: {args.skip_tests}')
+    logger.info(f'fail_fast: {args.fail_fast}')
     skip_tests = args.skip_tests == "true"
     fail_fast = args.fail_fast == "true"
     main(skip_tests, fail_fast)
diff --git a/python-frontend/typeshed_serializer/serializer/serializers.py b/python-frontend/typeshed_serializer/serializer/serializers.py
@@ -66,8 +66,19 @@ def walk_typeshed_third_parties(opt: options.Options = get_options()):
     build_result = build.build(source_list, opt)
     return build_result, source_paths
 
+def get_fq_module_name(file, package_name):
+    module_name = file.replace(".pyi", "")
+    fq_module_name = (
+        f"{package_name}.{module_name}" if package_name != "" else module_name
+    )
+    if module_name == "__init__":
+        fq_module_name = package_name
+    return fq_module_name
+
+def do_nothing_file_filter(root, file, file_path, fq_module_name):
+    return True
 
-def get_sources(relative_path: str):
+def get_sources(relative_path: str, file_filter = do_nothing_file_filter):
     source_list = []
     source_paths = set()
     path = os.path.join(CURRENT_PATH, relative_path)
@@ -82,13 +93,11 @@ def get_sources(relative_path: str):
             if not file.endswith(".pyi"):
                 # Only consider actual stubs
                 continue
-            module_name = file.replace(".pyi", "")
-            fq_module_name = (
-                f"{package_name}.{module_name}" if package_name != "" else module_name
-            )
-            if module_name == "__init__":
-                fq_module_name = package_name
             file_path = f"{root}/{file}"
+            fq_module_name = get_fq_module_name(file, package_name)
+            if not file_filter(root, file, file_path, fq_module_name):
+                continue
+
             source = build.BuildSource(file_path, module=fq_module_name)
             source_list.append(source)
             source_paths.add(source.path)
@@ -225,10 +234,15 @@ class MicrosoftStubsSerializer(Serializer):
     output_folder = f"{FolderManager.output_folder}/{save_location}"
 
     def get_build_result(self, opt=get_options()):
-        src_list, src_paths = get_sources(SKLEARN_STUBS_PATH)
+        src_list, src_paths = get_sources(SKLEARN_STUBS_PATH, self.filter_get_sources_result)
+
         build_result = build.build(src_list, opt)
         return build_result, src_paths
 
+
+    def filter_get_sources_result(self, root, file, file_path, fq_module_name):
+        return "matplotlib" not in fq_module_name
+
     def is_exception(self, file, build_result, source_paths):
         file_path = build_result.files[file].path
         # Filtering out KDTree and BallTree to avoid FPs
diff --git a/python-frontend/typeshed_serializer/tests/test_serializers.py b/python-frontend/typeshed_serializer/tests/test_serializers.py
@@ -17,6 +17,7 @@
 
 import logging
 import os
+import tempfile
 from unittest.mock import Mock, patch
 
 from serializer import symbols, serializers
@@ -25,6 +26,7 @@
     MicrosoftStubsSerializer,
     TypeshedSerializer,
     ImporterSerializer,
+    get_sources,
 )
 from tests import conftest
 from tests.conftest import MOCK_THIRD_PARTY_STUBS_LIST
@@ -72,7 +74,7 @@ def test_custom_stubs_serializer(typeshed_custom_stubs):
         custom_stubs_serializer.serialize()
         assert custom_stubs_serializer.get_build_result.call_count == 1
         # Not every files from "typeshed_custom_stubs" build are serialized, as some are builtins
-        assert symbols.save_module.call_count == 159
+        assert symbols.save_module.call_count == 162
 
 
 def test_importer_serializer():
@@ -125,3 +127,46 @@ def test_save_merged_symbols():
         ts_serializer.serialize_merged_modules()
         assert symbols.save_module.call_count == 1
         assert symbols.save_module.mock_calls[0].args[0] == merged_module_symbol
+
+def test_get_sources():
+    with tempfile.TemporaryDirectory() as temp_dir:
+        file_test_structure = {
+            "module1.pyi": "# Module 1 stub",
+            "module2.pyi": "# Module 2 stub", 
+            "regular_file.py": "# Not a stub file",
+            "text_file.txt": "# Text file",
+            "subpackage/__init__.pyi": "# Subpackage init",
+            "subpackage/submodule.pyi": "# Submodule stub",
+            "subpackage/python2/old_module.pyi": "# Python 2 stub (should be excluded)",
+            "excluded_module.pyi": "# Module to be excluded by filter",
+        }
+        
+        for file_path, content in file_test_structure.items():
+            full_path = os.path.join(temp_dir, file_path)
+            os.makedirs(os.path.dirname(full_path), exist_ok=True)
+            with open(full_path, 'w') as f:
+                f.write(content)
+                 
+        source_list, source_paths = get_sources(temp_dir)
+        assert len(source_list) == 5
+        assert len(source_paths) == 5
+
+        module_names = [source.module for source in source_list]
+        expected_modules = {
+            "module1", "module2", "excluded_module", "subpackage", "subpackage.submodule"
+        }
+        assert set(module_names) == expected_modules
+        
+        def exclude_filter(root, file_str, file_path, fq_module_name):
+            return "excluded" not in file_str and "excluded" not in fq_module_name
+        
+        source_list_filtered, source_paths_filtered = get_sources(temp_dir, exclude_filter)
+        
+        assert len(source_list_filtered) == 4
+        assert len(source_paths_filtered) == 4
+        
+        filtered_module_names = [source.module for source in source_list_filtered]
+        expected_filtered_modules = {
+            "module1", "module2", "subpackage", "subpackage.submodule"
+        }
+        assert set(filtered_module_names) == expected_filtered_modules

-Original file line number
+Diff line change
@@ @@ -0,0 +1,15 @@ @@
++
 +boto3x
 +clientboto3.client"8
 +botocore.client.BaseClient"botocore.client.BaseClient*
 +args
 +Any*
 +kwargs
 +Any*f
 +__path__boto3.__path__J
 +builtins.list[builtins.str]
 +builtins.str"builtins.str"builtins.list*�
 +__annotations__boto3.__annotations__W
 +builtins.dict[builtins.str,Any]
 +builtins.str"builtins.str
 +Any"builtins.dict
-Original file line number
+Diff line change
@@ @@ -0,0 +1,18 @@ @@
++
 +botocore.client�
++
 +BaseClientbotocore.client.BaseClient"*SonarPythonAnalyzerFakeStub.CustomStubBase*�
 +invoke!botocore.client.BaseClient.invoke"
 +Any*B
 +self8
 +botocore.client.BaseClient"botocore.client.BaseClient*.
 +FunctionName
 +builtins.str"builtins.str*0
 +InvocationType
 +builtins.str"builtins.str*)
 +Payload
 +builtins.str"builtins.str*�
 +__annotations__botocore.client.__annotations__W
 +builtins.dict[builtins.str,Any]
 +builtins.str"builtins.str
 +Any"builtins.dict
-Original file line number
+Diff line change
@@ @@ -0,0 +1,10 @@ @@
++
 +importlib_metadata._typing*�
 +__annotations__*importlib_metadata._typing.__annotations__W
 +builtins.dict[builtins.str,Any]
 +builtins.str"builtins.str
 +Any"builtins.dict*�
 +md_none"importlib_metadata._typing.md_noneu
 +functools.partial[typing._T]B
 +	typing._T"
 +builtins.object"builtins.object"builtins.object"functools.partial