feat: add max_tokens configuration to LLMConfig and LLMClient with validation

Author: duartebarbosadev

app/llm_provider.py

@@ -18,6 +18,7 @@
 
 logger = logging.getLogger(__name__)
 DEFAULT_LLM_TIMEOUT_SECONDS = 60
+DEFAULT_LLM_MAX_TOKENS = 4096
 DEFAULT_LLM_RETRIES = 2
 DEFAULT_STRUCTURED_OUTPUT_RETRIES = 1
 
@@ -120,13 +121,16 @@ class LLMConfig:
     site_name: Optional[str] = None
     send_site_info: bool = True
     timeout_seconds: int = DEFAULT_LLM_TIMEOUT_SECONDS
+    max_tokens: int = DEFAULT_LLM_MAX_TOKENS
     num_retries: int = DEFAULT_LLM_RETRIES
     structured_output_retries: int = DEFAULT_STRUCTURED_OUTPUT_RETRIES
 
     def __post_init__(self):
         """Validate configuration after initialization."""
         if not self.model:
             raise ValueError("Model name is required")
+        if self.max_tokens <= 0:
+            raise ValueError("Max tokens must be greater than zero")
         if self.num_retries < 0:
             raise ValueError("Number of retries cannot be negative")
         if self.structured_output_retries < 0:
@@ -241,7 +245,7 @@ def chat_completion(
             "custom_llm_provider": self.config.provider,
             "messages": messages,
             "temperature": temperature,
-            "max_tokens": kwargs.pop("max_tokens", 4096),
+            "max_tokens": kwargs.pop("max_tokens", self.config.max_tokens),
             "timeout": kwargs.pop("timeout", self.config.timeout_seconds),
             "num_retries": kwargs.pop("num_retries", self.config.num_retries),
             **kwargs,

app/tests/test_translation.py

@@ -656,8 +656,70 @@ def test_llm_client_accepts_fenced_raw_string_batch_json(self):
             [("hello", "Hola"), ("goodbye", "Adiós")],
         )
         self.assertEqual(mock_completion.call_args.kwargs["timeout"], 60)
+        self.assertEqual(mock_completion.call_args.kwargs["max_tokens"], 2048)
         self.assertEqual(mock_completion.call_args.kwargs["num_retries"], 2)
 
+    def test_llm_client_uses_configured_max_tokens(self):
+        """Configured max tokens should be used when no per-call override is provided."""
+
+        response = SimpleNamespace(
+            choices=[
+                SimpleNamespace(
+                    message=SimpleNamespace(
+                        content='{"translations": [{"key": "hello", "translation": "Hola"}]}',
+                        reasoning_content=None,
+                    )
+                )
+            ]
+        )
+        llm_config = LLMConfig(
+            provider="openrouter",
+            model="openrouter/owl-alpha",
+            max_tokens=1234,
+        )
+
+        with patch(
+            "llm_provider.litellm.completion", return_value=response
+        ) as mock_completion:
+            LLMClient(llm_config).chat_completion(
+                messages=[],
+                response_model=StringBatchTranslation,
+                temperature=0,
+            )
+
+        self.assertEqual(mock_completion.call_args.kwargs["max_tokens"], 1234)
+
+    def test_llm_client_allows_max_tokens_override(self):
+        """Callers can override max tokens per request."""
+
+        response = SimpleNamespace(
+            choices=[
+                SimpleNamespace(
+                    message=SimpleNamespace(
+                        content='{"translations": [{"key": "hello", "translation": "Hola"}]}',
+                        reasoning_content=None,
+                    )
+                )
+            ]
+        )
+        llm_config = LLMConfig(
+            provider="openrouter",
+            model="openrouter/owl-alpha",
+            max_tokens=1234,
+        )
+
+        with patch(
+            "llm_provider.litellm.completion", return_value=response
+        ) as mock_completion:
+            LLMClient(llm_config).chat_completion(
+                messages=[],
+                response_model=StringBatchTranslation,
+                temperature=0,
+                max_tokens=512,
+            )
+
+        self.assertEqual(mock_completion.call_args.kwargs["max_tokens"], 512)
+
     def test_llm_client_allows_retry_override(self):
         """Callers can override the default LiteLLM retry count per request."""
 
@@ -697,6 +759,16 @@ def test_llm_config_rejects_negative_retries(self):
                 num_retries=-1,
             )
 
+    def test_llm_config_rejects_non_positive_max_tokens(self):
+        """Max tokens must be positive because providers reject invalid caps."""
+
+        with self.assertRaisesRegex(ValueError, "Max tokens must be greater than zero"):
+            LLMConfig(
+                provider="openrouter",
+                model="openrouter/owl-alpha",
+                max_tokens=0,
+            )
+
     def test_llm_client_retries_invalid_structured_output(self):
         """Malformed model JSON should trigger a fresh structured-output attempt."""
 

requirements.txt

@@ -5,3 +5,4 @@ babel>=2.17.0
 ruff>=0.8.0
 pydantic>=2.0.0
 litellm>=1.60.0
+tenacity>=8.0.0