feat: add comprehensive type checking with mypy strict mode and pyright (#20)

- Upgrade mypy to strict mode with additional error codes (ignore-without-code,
  redundant-cast, truthy-bool, truthy-iterable, unused-awaitable)
- Add pyright as a second type checker for comprehensive coverage
- Configure pyright in strict mode with practical exceptions for
  third-party libraries lacking full stubs
- Update CI to run both type checkers in parallel via matrix strategy
- Fix CompletedProcess generic type parameter in cli.py
- Add type: ignore comments for untyped posthog methods in telemetry.py
- Update AGENTS.md documentation with new type checking setup

The dual type checker approach catches different categories of issues:
- mypy: The standard Python type checker, excellent for protocol validation
- pyright: Microsoft's type checker, catches additional edge cases

Both run in strict mode to enforce rigorous typing standards.

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: mldangelo

.github/workflows/test.yml

@@ -42,9 +42,12 @@ jobs:
         run: uv run ruff format --check src/
 
   type-check:
-    name: Type Check
+    name: Type Check (${{ matrix.type-checker }})
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    strategy:
+      matrix:
+        type-checker: [mypy, pyright]
     steps:
       - uses: actions/checkout@v6
 
@@ -59,8 +62,13 @@ jobs:
         run: uv sync --extra dev
 
       - name: Type check with mypy
+        if: matrix.type-checker == 'mypy'
         run: uv run mypy src/promptfoo/
 
+      - name: Type check with pyright
+        if: matrix.type-checker == 'pyright'
+        run: uv run pyright src/promptfoo/
+
   test:
     name: Test (py${{ matrix.python-version }}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}

AGENTS.md

@@ -134,7 +134,9 @@ Runs on every PR and push to main:
 
 - **Lint**: Ruff linting (`uv run ruff check src/`)
 - **Format Check**: Ruff formatting (`uv run ruff format --check src/`)
-- **Type Check**: mypy static analysis (`uv run mypy src/promptfoo/`)
+- **Type Check**: Both mypy and pyright in strict mode (run in parallel via matrix)
+  - `uv run mypy src/promptfoo/` - Standard Python type checker
+  - `uv run pyright src/promptfoo/` - Microsoft's type checker for additional coverage
 - **Unit Tests**: Fast tests with mocked dependencies (`uv run pytest -m 'not smoke'`)
 - **Smoke Tests**: Integration tests against real CLI (`uv run pytest tests/smoke/`)
 - **Build**: Package build validation
@@ -180,7 +182,9 @@ We use **OpenID Connect (OIDC)** for secure, credential-free PyPI publishing:
 
 - **Linter**: Ruff with extended rule sets (isort, pycodestyle, flake8-bugbear, etc.)
 - **Formatter**: Ruff (replaces Black)
-- **Type Checker**: mypy with strict settings
+- **Type Checkers**: Both **mypy** and **pyright** in strict mode for comprehensive coverage
+  - **mypy**: The standard Python type checker with strict mode and additional error codes
+  - **pyright**: Microsoft's fast type checker that catches different issues than mypy
 - **Package Manager**: uv (Astral's fast Python package manager)
 
 ### Running Checks Locally
@@ -198,9 +202,15 @@ uv run ruff check src/ --fix
 # Format code
 uv run ruff format src/
 
-# Type check
+# Type check with mypy (strict mode)
 uv run mypy src/promptfoo/
 
+# Type check with pyright (strict mode)
+uv run pyright src/promptfoo/
+
+# Run both type checkers (recommended before PR)
+uv run mypy src/promptfoo/ && uv run pyright src/promptfoo/
+
 # Run tests
 uv run pytest
 ```
@@ -330,6 +340,7 @@ git checkout -b feat/my-feature-name
 uv run ruff check src/ --fix
 uv run ruff format src/
 uv run mypy src/promptfoo/
+uv run pyright src/promptfoo/
 uv run pytest
 
 # 4. Commit with conventional commit message
@@ -357,6 +368,7 @@ git checkout -b fix/bug-description
 uv run ruff check src/ --fix
 uv run ruff format src/
 uv run mypy src/promptfoo/
+uv run pyright src/promptfoo/
 uv run pytest
 
 # 4. Commit with conventional commit message

pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
 dev = [
     "pytest>=8.4.0",
     "mypy>=1.16.0",
+    "pyright>=1.1.400",
     "ruff>=0.12.0",
     "types-pyyaml>=6.0.0",
 ]
@@ -92,16 +93,64 @@ quote-style = "double"
 
 [tool.mypy]
 python_version = "3.9"
-warn_return_any = true
-warn_unused_configs = true
-warn_redundant_casts = true
+# Enable strict mode for comprehensive type checking
+strict = true
+# Additional strictness beyond --strict
 warn_unreachable = true
-no_implicit_optional = true
-strict_equality = true
+enable_error_code = [
+    "ignore-without-code",
+    "redundant-cast",
+    "truthy-bool",
+    "truthy-iterable",
+    "unused-awaitable",
+]
+# Output formatting
 show_error_codes = true
+show_column_numbers = true
 pretty = true
-check_untyped_defs = true
-disallow_incomplete_defs = true
+# Error handling
+warn_unused_ignores = true
+
+[[tool.mypy.overrides]]
+module = "posthog.*"
+ignore_missing_imports = true
+
+[tool.pyright]
+pythonVersion = "3.9"
+pythonPlatform = "All"
+typeCheckingMode = "strict"
+include = ["src/promptfoo"]
+exclude = ["tests", "**/__pycache__"]
+# Report all errors in strict mode
+reportMissingImports = true
+reportMissingTypeStubs = false  # Disable for posthog which lacks full stubs
+reportUnusedImport = true
+reportUnusedClass = true
+reportUnusedFunction = true
+reportUnusedVariable = true
+reportDuplicateImport = true
+reportPrivateUsage = true
+reportConstantRedefinition = true
+reportIncompatibleMethodOverride = true
+reportIncompatibleVariableOverride = true
+reportInconsistentConstructor = true
+reportOverlappingOverload = true
+reportUninitializedInstanceVariable = true
+reportCallInDefaultInitializer = true
+reportUnnecessaryIsInstance = true
+reportUnnecessaryCast = true
+reportUnnecessaryComparison = true
+reportUnnecessaryContains = true
+reportImplicitStringConcatenation = false
+reportUnusedCallResult = false
+reportUnusedExpression = true
+reportUnnecessaryTypeIgnoreComment = false  # Allow type: ignore for mypy compatibility
+reportMatchNotExhaustive = true
+# Relax some strict checks for third-party libraries without full stubs
+reportUnknownMemberType = false
+reportUnknownArgumentType = false
+reportUnknownVariableType = false
+reportUnknownParameterType = false
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

src/promptfoo/cli.py

@@ -165,7 +165,7 @@ def _requires_shell(executable: str) -> bool:
     return ext.lower() in _WINDOWS_SHELL_EXTENSIONS
 
 
-def _run_command(cmd: list[str], env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess:
+def _run_command(cmd: list[str], env: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess[bytes]:
     """Execute a command, handling shell requirements on Windows."""
     if _requires_shell(cmd[0]):
         return subprocess.run(subprocess.list2cmdline(cmd), shell=True, env=env)

src/promptfoo/telemetry.py

@@ -172,8 +172,8 @@ def shutdown(self) -> None:
         """Shutdown the telemetry client and flush any pending events."""
         if self._client:
             try:
-                self._client.flush()
-                self._client.shutdown()
+                self._client.flush()  # type: ignore[no-untyped-call]
+                self._client.shutdown()  # type: ignore[no-untyped-call]
             except Exception:
                 pass  # Silently fail
             finally: