feat: partially implement optimize_from_config

Author: andrewklatzke

packages/optimization/src/ldai_optimization/__init__.py

@@ -13,12 +13,14 @@
     OptimizationOptions,
     ToolDefinition,
 )
+from ldai_optimization.ld_api_client import LDApiError
 
 __version__ = "0.0.0"
 
 __all__ = [
     '__version__',
     'AIJudgeCallConfig',
+    'LDApiError',
     'OptimizationClient',
     'OptimizationContext',
     'OptimizationFromConfigOptions',

packages/optimization/src/ldai_optimization/client.py

@@ -45,6 +45,21 @@
 
 logger = logging.getLogger(__name__)
 
+
+def _strip_provider_prefix(model: str) -> str:
+    """Strip the provider prefix from a model identifier returned by the LD API.
+
+    API model keys are formatted as "Provider.model-name" (e.g. "OpenAI.gpt-5",
+    "Anthropic.claude-opus-4.6"). Only the part after the first period is needed
+    by the underlying LLM clients. If no period is present the string is returned
+    unchanged.
+
+    :param model: Raw model string from the API.
+    :return: Model name with provider prefix removed.
+    """
+    return model.split(".", 1)[-1]
+
+
 # Maps SDK status strings to the API status/activity values expected by
 # agent_optimization_result records. Defined at module level to avoid
 # allocating the dict on every on_status_update invocation.
@@ -981,10 +996,12 @@ def _build_options_from_config(
                 judge_key=judge["key"],
             )
 
-        if not judges and options.on_turn is None:
+        has_ground_truth = bool(config.get("groundTruthResponses"))
+        if not judges and not has_ground_truth and options.on_turn is None:
             raise ValueError(
-                "The optimization config has no acceptance statements or judges, "
-                "and no on_turn callback was provided. At least one is required."
+                "The optimization config has no acceptance statements, judges, or ground truth "
+                "responses, and no on_turn callback was provided. At least one is required to "
+                "evaluate optimization results."
             )
 
         variable_choices: List[Dict[str, Any]] = config["variableChoices"] or [{}]
@@ -1034,8 +1051,8 @@ def _persist_and_forward(
         return OptimizationOptions(
             context_choices=options.context_choices,
             max_attempts=config["maxAttempts"],
-            model_choices=config["modelChoices"],
-            judge_model=config["judgeModel"],
+            model_choices=[_strip_provider_prefix(m) for m in config["modelChoices"]],
+            judge_model=_strip_provider_prefix(config["judgeModel"]),
             variable_choices=variable_choices,
             handle_agent_call=options.handle_agent_call,
             handle_judge_call=options.handle_judge_call,

packages/optimization/src/ldai_optimization/ld_api_client.py

@@ -10,6 +10,28 @@
 
 _BASE_URL = "https://app.launchdarkly.com"
 
+
+class LDApiError(Exception):
+    """Raised when the LaunchDarkly REST API returns an error or is unreachable.
+
+    Attributes:
+        status_code: HTTP status code, or None for network-level failures.
+        path: The API path that was requested.
+    """
+
+    def __init__(self, message: str, status_code: Optional[int] = None, path: str = "") -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.path = path
+
+
+_HTTP_ERROR_HINTS: Dict[int, str] = {
+    401: "Authentication failed — check that LAUNCHDARKLY_API_KEY is set correctly.",
+    403: "Authorization failed — check that your API key has the required permissions.",
+    404: "Resource not found — check that the project key and optimization config key are correct.",
+    429: "Rate limit exceeded — too many requests to the LaunchDarkly API.",
+}
+
 _REQUIRED_STRING_FIELDS = ("id", "key", "aiConfigKey", "judgeModel")
 _REQUIRED_INT_FIELDS = ("maxAttempts", "version", "createdAt")
 _REQUIRED_LIST_FIELDS = (
@@ -170,12 +192,18 @@ def _request(self, method: str, path: str, body: Any = None) -> Any:
                 return json.loads(raw) if raw else None
         except urllib.error.HTTPError as exc:
             body_excerpt = exc.read(500).decode(errors="replace")
-            raise RuntimeError(
-                f"LaunchDarkly API error {exc.code} for {method} {path}: {body_excerpt}"
+            hint = _HTTP_ERROR_HINTS.get(exc.code, "")
+            detail = f"{hint} (API response: {body_excerpt})" if hint else f"API response: {body_excerpt}"
+            raise LDApiError(
+                f"LaunchDarkly API error {exc.code} {exc.msg} for {method} {path}. {detail}",
+                status_code=exc.code,
+                path=path,
             ) from exc
         except urllib.error.URLError as exc:
-            raise RuntimeError(
-                f"LaunchDarkly API request failed for {method} {path}: {exc.reason}"
+            raise LDApiError(
+                f"Could not reach LaunchDarkly API at {url}: {exc.reason}. "
+                "Check your network connection and the base_url setting.",
+                path=path,
             ) from exc
 
     def get_agent_optimization(
@@ -186,7 +214,7 @@ def get_agent_optimization(
         :param project_key: LaunchDarkly project key.
         :param optimization_key: Key of the agent optimization config.
         :return: Validated AgentOptimizationConfig.
-        :raises RuntimeError: On non-200 HTTP responses or network errors.
+        :raises LDApiError: On non-200 HTTP responses or network errors.
         :raises ValueError: If the response is missing required fields.
         """
         path = f"/api/v2/projects/{project_key}/agent-optimizations/{optimization_key}"
@@ -208,7 +236,17 @@ def post_agent_optimization_result(
         path = f"/api/v2/projects/{project_key}/agent-optimizations/{optimization_id}/results"
         try:
             self._request("POST", path, body=payload)
-        except Exception:
-            logger.exception(
-                "Failed to persist optimization result for optimization_id=%s", optimization_id
+        except LDApiError as exc:
+            logger.debug(
+                "Failed to persist optimization result (optimization_id=%s, iteration=%s): %s",
+                optimization_id,
+                payload.get("iteration"),
+                exc,
+            )
+        except Exception as exc:
+            logger.debug(
+                "Unexpected error persisting optimization result (optimization_id=%s, iteration=%s): %s",
+                optimization_id,
+                payload.get("iteration"),
+                exc,
             )

packages/optimization/tests/test_client.py

@@ -1105,9 +1105,16 @@ def test_acceptance_statements_and_judges_merged(self):
         assert "acceptance-statement-0" in result.judges
         assert "accuracy" in result.judges
 
-    def test_raises_when_no_judges_and_no_on_turn(self):
+    def test_raises_when_no_judges_no_ground_truth_no_on_turn(self):
         config = dict(_API_CONFIG, acceptanceStatements=[], judges=[])
-        with pytest.raises(ValueError, match="no acceptance statements or judges"):
+        with pytest.raises(ValueError, match="no acceptance statements, judges, or ground truth"):
+            self._build(config=config)
+
+    def test_ground_truth_responses_alone_does_not_pass_no_criteria_check(self):
+        # groundTruthResponses is not yet implemented as standalone criteria;
+        # OptimizationOptions still requires judges or on_turn.
+        config = dict(_API_CONFIG, acceptanceStatements=[], judges=[], groundTruthResponses=["4"])
+        with pytest.raises((ValueError, Exception)):
             self._build(config=config)
 
     def test_on_turn_satisfies_no_judges_requirement(self):
@@ -1138,14 +1145,29 @@ def test_max_attempts_from_config(self):
         result = self._build()
         assert result.max_attempts == 3
 
-    def test_model_choices_from_config(self):
+    def test_model_choices_provider_prefix_stripped(self):
+        config = dict(_API_CONFIG, modelChoices=["OpenAI.gpt-4o", "Anthropic.claude-opus-4-5"])
+        result = self._build(config=config)
+        assert result.model_choices == ["gpt-4o", "claude-opus-4-5"]
+
+    def test_judge_model_provider_prefix_stripped(self):
+        config = dict(_API_CONFIG, judgeModel="OpenAI.gpt-4o")
+        result = self._build(config=config)
+        assert result.judge_model == "gpt-4o"
+
+    def test_model_choices_without_prefix_unchanged(self):
         result = self._build()
         assert result.model_choices == ["gpt-4o", "gpt-4o-mini"]
 
-    def test_judge_model_from_config(self):
+    def test_judge_model_without_prefix_unchanged(self):
         result = self._build()
         assert result.judge_model == "gpt-4o"
 
+    def test_model_with_multiple_dots_only_prefix_stripped(self):
+        config = dict(_API_CONFIG, judgeModel="Anthropic.claude-opus-4.6")
+        result = self._build(config=config)
+        assert result.judge_model == "claude-opus-4.6"
+
     def test_callbacks_forwarded_from_options(self):
         handle_agent = AsyncMock(return_value="ok")
         handle_judge = AsyncMock(return_value=JUDGE_PASS_RESPONSE)

packages/optimization/tests/test_ld_api_client.py

@@ -12,6 +12,7 @@
 from ldai_optimization.ld_api_client import (
     AgentOptimizationConfig,
     LDApiClient,
+    LDApiError,
     OptimizationResultPayload,
     _parse_agent_optimization,
 )
@@ -160,20 +161,42 @@ def test_authorization_header_always_sent(self):
             req: urllib.request.Request = mock_open.call_args[0][0]
             assert req.get_header("Authorization") == "my-api-key"
 
-    def test_raises_runtime_error_on_http_error(self):
+    def test_raises_ld_api_error_on_http_error(self):
         client = LDApiClient("test-key")
         http_error = urllib.error.HTTPError(
             url="http://x", code=404, msg="Not Found", hdrs=MagicMock(), fp=BytesIO(b"not found body")
         )
         with patch("urllib.request.urlopen", side_effect=http_error):
-            with pytest.raises(RuntimeError, match="404"):
+            with pytest.raises(LDApiError) as exc_info:
                 client._request("GET", "/missing")
+        assert exc_info.value.status_code == 404
+        assert "404" in str(exc_info.value)
 
-    def test_raises_runtime_error_on_url_error(self):
+    def test_raises_ld_api_error_on_url_error(self):
         client = LDApiClient("test-key")
         url_error = urllib.error.URLError(reason="Connection refused")
         with patch("urllib.request.urlopen", side_effect=url_error):
-            with pytest.raises(RuntimeError, match="Connection refused"):
+            with pytest.raises(LDApiError) as exc_info:
+                client._request("GET", "/path")
+        assert exc_info.value.status_code is None
+        assert "Connection refused" in str(exc_info.value)
+
+    def test_401_error_includes_api_key_hint(self):
+        client = LDApiClient("test-key")
+        http_error = urllib.error.HTTPError(
+            url="http://x", code=401, msg="Unauthorized", hdrs=MagicMock(), fp=BytesIO(b"")
+        )
+        with patch("urllib.request.urlopen", side_effect=http_error):
+            with pytest.raises(LDApiError, match="LAUNCHDARKLY_API_KEY"):
+                client._request("GET", "/path")
+
+    def test_404_error_includes_key_hint(self):
+        client = LDApiClient("test-key")
+        http_error = urllib.error.HTTPError(
+            url="http://x", code=404, msg="Not Found", hdrs=MagicMock(), fp=BytesIO(b"")
+        )
+        with patch("urllib.request.urlopen", side_effect=http_error):
+            with pytest.raises(LDApiError, match="project key"):
                 client._request("GET", "/path")
 
     def test_custom_base_url_used_in_request(self):
@@ -218,14 +241,15 @@ def test_raises_on_invalid_response(self):
             with pytest.raises(ValueError, match="Invalid AgentOptimization response"):
                 client.get_agent_optimization("proj", "opt")
 
-    def test_raises_runtime_error_on_http_404(self):
+    def test_raises_ld_api_error_on_http_404(self):
         client = LDApiClient("test-key")
         http_error = urllib.error.HTTPError(
             url="http://x", code=404, msg="Not Found", hdrs=MagicMock(), fp=BytesIO(b"not found")
         )
         with patch("urllib.request.urlopen", side_effect=http_error):
-            with pytest.raises(RuntimeError, match="404"):
+            with pytest.raises(LDApiError) as exc_info:
                 client.get_agent_optimization("proj", "missing-key")
+        assert exc_info.value.status_code == 404
 
 
 # ---------------------------------------------------------------------------