diff --git a/build_tools/analysis/README.md b/build_tools/analysis/README.md index bb8c02408..06fc5d557 100644 --- a/build_tools/analysis/README.md +++ b/build_tools/analysis/README.md @@ -223,6 +223,36 @@ According to PEP 561 and the Python typing documentation, a library is considere 4. Generic types are properly parameterized 5. The library passes type checking with mypy in strict mode +**Exemptions from Type Annotation Requirements:** + +The analyzer implements the following exemptions as specified in the type completeness guidelines: + +1. **Constants with simple literal values** - Constants assigned simple literals don't require annotations + - The variable name must be in ALL_CAPS (including private constants like `_MAX_VALUE`) + - The assigned value must be a "simple literal": str, int, float, bool, None, or containers (list, tuple, dict, set) composed only of such simple literals + - Example exempt constants: `MAX_VALUE = 100`, `DEFAULT_NAME = "test"`, `ALLOWED = ["a", "b", "c"]` + - Complex values containing non-simple literals still require annotations: `BYTES_VAL: bytes = b"data"` + - Note: The analyzer does not currently check for `Final` annotations when deciding exemptions + +2. **Enum values** - Values within an Enum class don't require annotations + - They automatically take on the type of the Enum class + +3. **Type aliases** - Type aliases don't require `TypeAlias` annotation + - Example: `MyType = dict[str, int]`, `Foo = Callable[[int], str]` + - These are recognized as type aliases by their assignment pattern + +4. **self and cls parameters** - Don't require explicit annotations + - Standard Python convention + +5. **__init__ return types** - Don't need annotation + - Always return None by definition + +6. **Special module-level symbols** - Don't require annotations + - `__all__`, `__author__`, `__copyright__`, `__email__`, `__license__`, `__title__`, `__uri__`, `__version__` + +7. **Special class-level symbols** - Don't require annotations + - `__class__`, `__dict__`, `__doc__`, `__module__`, `__slots__` + ### Analysis Categories **Type Hint Status:** @@ -235,12 +265,15 @@ According to PEP 561 and the Python typing documentation, a library is considere - **Functions/Methods:** Function signatures including parameters and return types - Excludes `self` and `cls` parameters from parameter counts + - `__init__` methods don't require return type annotations (always None) - Considers both parameters and return type for completeness - Tracks decorator usage (e.g., `@staticmethod`, `@lru_cache`) - **Class Variables:** Variables defined at class level - Distinguishes between annotated (`class_var: int = 10`) and non-annotated - Can include `ClassVar` type hints for class-specific attributes + - Special symbols like `__slots__` are exempt from annotation requirements + - Enum values are exempt (they inherit the Enum class type) - **Instance Variables:** Variables defined as instance attributes (e.g., `self.attr`) - Detected in `__init__` and other methods @@ -248,11 +281,14 @@ According to PEP 561 and the Python typing documentation, a library is considere - **Module Variables:** Variables defined at module level - Includes constants, configuration values, and exported names + - Constants with simple literal values (ALL_CAPS naming) are exempt + - Special symbols like `__all__`, `__version__` are exempt from annotation requirements - Important for library API clarity - **Type Aliases:** Type alias definitions (using TypeAlias annotation) - Modern syntax: `MyType: TypeAlias = dict[str, int]` - Also detects `typing.TypeAlias` and `typing_extensions.TypeAlias` + - Type aliases without explicit `TypeAlias` annotation are also recognized and exempt **Priority Levels:** diff --git a/build_tools/analysis/type_hint_analyzer.py b/build_tools/analysis/type_hint_analyzer.py index e1dce96fd..b0ffe98dc 100644 --- a/build_tools/analysis/type_hint_analyzer.py +++ b/build_tools/analysis/type_hint_analyzer.py @@ -11,6 +11,19 @@ Based on the type completeness information from https://typing.python.org/en/latest/guides/libraries.html#type-completeness +The analyzer implements the following exemptions from type annotation requirements: + +1. Constants assigned simple literal values (e.g., RED = '#F00', MAX_TIMEOUT = 50) + - Named in ALL_CAPS (including private constants like _MAX_VALUE) + - Assigned a simple literal value (str, int, float, bool, None, or containers of these) +2. Enum values within an Enum class +3. Type aliases (assignments to instantiable types) +4. self/cls parameters in methods +5. __init__ return types (always None) +6. Module-level symbols: __all__, __author__, __copyright__, __email__, + __license__, __title__, __uri__, __version__ +7. Class-level symbols: __class__, __dict__, __doc__, __module__, __slots__ + Usage: python type_hint_analyzer.py [--output-dir OUTPUT_DIR] @@ -27,6 +40,17 @@ from pathlib import Path from typing import Any, Dict, List, Tuple +# Constants for type checking +_NONE_TYPE = type(None) +_TYPE_ALIAS_KEYWORDS = { + "Optional", + "Union", + "Callable", + "Type", + "Any", + "NoReturn", +} + def count_mypy_errors_by_submodule(pythainlp_dir: str) -> Dict[str, int]: """ @@ -51,8 +75,8 @@ def count_mypy_errors_by_submodule(pythainlp_dir: str) -> Dict[str, int]: submodule_path = os.path.join(pythainlp_dir, submodule) try: # Run mypy on the submodule - result = subprocess.run( - ["mypy", submodule_path, "--ignore-missing-imports"], + result = subprocess.run( # noqa: S603 + ["mypy", submodule_path, "--ignore-missing-imports"], # noqa: S607 capture_output=True, text=True, timeout=60, @@ -91,6 +115,7 @@ def __init__(self, filepath: str, module_path: str): self.current_class = None self.current_function = None self.module_level = True + self.current_class_bases = [] # Track base classes to detect Enum def is_private(self, name: str) -> bool: """Check if a name is private (starts with underscore).""" @@ -104,13 +129,19 @@ def is_public(self, name: str) -> bool: def check_function_type_hints( self, node: ast.FunctionDef - ) -> Tuple[str, int, int]: + ) -> Tuple[str, int, int, bool]: """ Check type hint completeness for a function. - Returns: (status, total_params, hinted_params) + + Returns: (status, total_params, hinted_params, has_return_hint) - status: "complete", "incomplete", "none" - total_params: number of parameters (excluding self/cls) - hinted_params: number of parameters with type hints + - has_return_hint: whether return type annotation is present + + According to type completeness guidelines: + - self/cls parameters don't require annotations + - __init__ methods don't require return type annotations """ # Count parameters (excluding self/cls) params = [] @@ -122,16 +153,35 @@ def check_function_type_hints( hinted_params = sum(1 for arg in params if arg.annotation is not None) has_return_hint = node.returns is not None + # __init__ methods don't need return type annotations + is_init = node.name == "__init__" + requires_return_hint = not is_init + # Determine status - if total_params == 0 and not has_return_hint: + if total_params == 0 and not requires_return_hint: + # No params and return type not required (e.g., __init__ with no params) + status = "complete" + elif total_params == 0 and requires_return_hint and not has_return_hint: + # No params but return type required and missing status = "none" elif total_params == 0 and has_return_hint: + # No params and has return type status = "complete" elif hinted_params == 0 and not has_return_hint: + # No hints at all status = "none" - elif hinted_params == total_params and has_return_hint: + elif requires_return_hint and ( + hinted_params == total_params and has_return_hint + ): + # All params and return type hinted + status = "complete" + elif ( + not requires_return_hint and hinted_params == total_params + ): + # All params hinted, return type not required (__init__) status = "complete" else: + # Partial hints status = "incomplete" return status, total_params, hinted_params, has_return_hint @@ -187,6 +237,211 @@ def _get_variable_name(self, target: ast.expr) -> str: else: return "unknown" + def _is_exempt_module_symbol(self, name: str) -> bool: + """ + Check if a symbol is exempt from type annotation requirements. + + According to type completeness guidelines, these module-level + symbols do not require type annotations: + __all__, __author__, __copyright__, __email__, __license__, + __title__, __uri__, __version__ + """ + exempt_symbols = { + "__all__", + "__author__", + "__copyright__", + "__email__", + "__license__", + "__title__", + "__uri__", + "__version__", + } + return name in exempt_symbols + + def _is_exempt_class_symbol(self, name: str) -> bool: + """ + Check if a class-level symbol is exempt from type annotations. + + According to type completeness guidelines, these class-level + symbols do not require type annotations: + __class__, __dict__, __doc__, __module__, __slots__ + """ + exempt_symbols = { + "__class__", + "__dict__", + "__doc__", + "__module__", + "__slots__", + } + return name in exempt_symbols + + def _is_simple_literal(self, value: ast.expr) -> bool: + """ + Check if a value is a simple literal. + + The value parameter must be an ast.expr node. + + Simple literals include: strings, numbers, booleans, None, + and simple containers (list, tuple, dict, set) containing + only simple literals. + """ + # Direct literal types (use ast.Constant for Python 3.8+) + # Only accept simple types: str, int, float, bool, None + if isinstance(value, ast.Constant): + return isinstance(value.value, (str, int, float, bool, _NONE_TYPE)) + + # Check for simple containers + if isinstance(value, (ast.List, ast.Tuple, ast.Set)): + return all(self._is_simple_literal(elt) for elt in value.elts) + + if isinstance(value, ast.Dict): + return all( + self._is_simple_literal(k) and self._is_simple_literal(v) + for k, v in zip(value.keys, value.values) + ) + + # Unary operations on literals (e.g., -1, +5) + if isinstance(value, ast.UnaryOp) and isinstance( + value.op, (ast.UAdd, ast.USub) + ): + return self._is_simple_literal(value.operand) + + return False + + def _is_constant_name(self, name: str) -> bool: + """ + Check if a variable name follows constant naming convention. + + Constants are typically named in ALL_CAPS (with optional underscores). + This includes both public (MAX_VALUE) and private (_MAX_VALUE) constants. + """ + # Check if name is in ALL_CAPS (with optional underscores) + # Remove leading underscores for the check + name_without_underscores = name.lstrip("_") + return bool( + name_without_underscores and name_without_underscores.isupper() + ) + + def _is_in_enum_class(self) -> bool: + """Check if currently inside an Enum class.""" + if not self.current_class: + return False + + # Check if any base class looks like Enum + for base in self.current_class_bases: + if isinstance(base, ast.Name) and "Enum" in base.id: + return True + elif isinstance(base, ast.Attribute) and "Enum" in base.attr: + return True + + return False + + def _looks_like_type_expr(self, expr: ast.expr) -> bool: + """ + Check if an expression looks like a type expression. + + This helps distinguish type unions (int | str) from bitwise operations (1 | 2). + """ + # Constant None (Python 3.8+) + if isinstance(expr, ast.Constant) and expr.value is None: + return True + + # Name that's a known type keyword or built-in type + if isinstance(expr, ast.Name): + # Known typing constructs + if expr.id in _TYPE_ALIAS_KEYWORDS: + return True + # Known built-in types (lowercase) + known_types = { + "int", "str", "float", "bool", "bytes", + "list", "dict", "set", "tuple", "frozenset", + } + if expr.id in known_types: + return True + # PEP 585 style types (capitalized versions) + pep_585_types = { + "List", "Dict", "Set", "Tuple", "FrozenSet", + } + if expr.id in pep_585_types: + return True + + # Subscripted type + if isinstance(expr, ast.Subscript): + return self._looks_like_type_expr(expr.value) + + # Attribute from typing module + if isinstance(expr, ast.Attribute): + if isinstance(expr.value, ast.Name): + if expr.value.id in ("typing", "typing_extensions"): + return True + + # Nested union (already has | operator) + if isinstance(expr, ast.BinOp) and isinstance(expr.op, ast.BitOr): + return self._looks_like_type_expr(expr.left) and self._looks_like_type_expr(expr.right) + + return False + + def _is_type_alias_without_annotation(self, value: ast.expr) -> bool: + """ + Check if an assignment is a type alias without TypeAlias annotation. + + Type aliases are assignments where the value is an instantiable type. + Examples: MyType = dict[str, int], Foo = Optional[str] + + This is conservative to avoid false positives like VALUE = mapping["key"] + or bitwise operations like FLAGS = FLAG_A | FLAG_B. + """ + if value is None: + return False + + # Check for common type alias patterns + # 1. Subscripted types: List[str], Dict[int, str], etc. + # Only if the base is a known type name or typing construct + if isinstance(value, ast.Subscript): + # Check if base is a Name and looks like a type + if isinstance(value.value, ast.Name): + base_name = value.value.id + # Common built-in types and typing constructs + known_types = { + "list", "dict", "set", "tuple", "frozenset", + "List", "Dict", "Set", "Tuple", "FrozenSet", + "Optional", "Union", "Callable", "Type", + "Sequence", "Mapping", "Iterable", "Iterator", + "Any", "Generic", "Protocol", "TypeVar", + } + # Base name starts with uppercase (type convention) or is a known type + if base_name in known_types or (base_name and base_name[0].isupper()): + return True + # Check if base is from typing module + elif isinstance(value.value, ast.Attribute): + if isinstance(value.value.value, ast.Name): + if value.value.value.id in ("typing", "typing_extensions"): + return True + return False + + # 2. Union types with | operator (Python 3.10+) + # Only if both operands look like type expressions + if isinstance(value, ast.BinOp) and isinstance(value.op, ast.BitOr): + # Check if both sides look like types (not bitwise flag operations) + if self._looks_like_type_expr(value.left) and self._looks_like_type_expr(value.right): + return True + return False + + # 3. Type names that suggest type aliases + if isinstance(value, ast.Name): + # Common typing constructs + if value.id in _TYPE_ALIAS_KEYWORDS: + return True + + # 4. Attribute access from typing module (typing.Something) + if isinstance(value, ast.Attribute): + # Only consider it a type alias if it's from typing/typing_extensions + if isinstance(value.value, ast.Name): + if value.value.id in ("typing", "typing_extensions"): + return True + + return False + def visit_AnnAssign(self, node: ast.AnnAssign): """Visit annotated assignment (variable with type hint).""" # Skip if we're in a function/method body (local variables) @@ -264,6 +519,44 @@ def visit_Assign(self, node: ast.Assign): for target in node.targets: var_name = self._get_variable_name(target) + # Check for exemptions based on type completeness guidelines + # Module-level exempt symbols + if self.module_level and self._is_exempt_module_symbol(var_name): + continue + + # Class-level exempt symbols + if ( + self.current_class is not None + and self.current_function is None + and self._is_exempt_class_symbol(var_name) + ): + continue + + # Enum values - skip if we're in an Enum class + if ( + self.current_class is not None + and self.current_function is None + and self._is_in_enum_class() + ): + # Enum values don't require annotations + continue + + # Constants with simple literal values don't require annotations + if ( + self.module_level + and self._is_constant_name(var_name) + and self._is_simple_literal(node.value) + ): + # This is an exempt constant + continue + + # Type aliases without annotation don't require annotations + if self.module_level and self._is_type_alias_without_annotation( + node.value + ): + # This is a type alias, doesn't need annotation + continue + # Determine variable type and qualified name if self._is_instance_variable(target): var_type = "instance_variable" @@ -374,11 +667,14 @@ def visit_ClassDef(self, node: ast.ClassDef): # Visit class body (methods and class variables) old_class = self.current_class + old_class_bases = self.current_class_bases self.current_class = node.name + self.current_class_bases = node.bases # Track base classes for Enum detection old_module_level = self.module_level self.module_level = False self.generic_visit(node) self.current_class = old_class + self.current_class_bases = old_class_bases self.module_level = old_module_level @@ -463,7 +759,7 @@ def count_references(qualified_name: str, all_files: List[str]) -> int: content = f.read() # Simple text search - not perfect but gives an approximation count += content.count(search_name) - except Exception: + except Exception: # noqa: S110 pass return count