diff --git a/notdiamond/llms/providers.py b/notdiamond/llms/providers.py index 8d6dda0b..b3530832 100644 --- a/notdiamond/llms/providers.py +++ b/notdiamond/llms/providers.py @@ -33,7 +33,6 @@ class NDLLMProviders(Enum): CLAUDE_2_1 (NDLLMProvider): refers to 'claude-2.1' model by Anthropic CLAUDE_3_OPUS_20240229 (NDLLMProvider): refers to 'claude-3-opus-20240229' model by Anthropic - CLAUDE_3_SONNET_20240229 (NDLLMProvider): refers to 'claude-3-sonnet-20240229' model by Anthropic CLAUDE_3_7_SONNET_LATEST (NDLLMProvider): refers to 'claude-3-7-sonnet-latest' model by Anthropic CLAUDE_3_7_SONNET_20250219 (NDLLMProvider): refers to 'claude-3-7-sonnet-20250219' model by Anthropic CLAUDE_3_5_HAIKU_20241022 (NDLLMProvider): refers to 'claude-3-5-haiku-20241022' model by Anthropic @@ -122,7 +121,6 @@ class NDLLMProviders(Enum): CLAUDE_2_1 = ("anthropic", "claude-2.1") CLAUDE_3_OPUS_20240229 = ("anthropic", "claude-3-opus-20240229") - CLAUDE_3_SONNET_20240229 = ("anthropic", "claude-3-sonnet-20240229") CLAUDE_3_7_SONNET_LATEST = ("anthropic", "claude-3-7-sonnet-latest") CLAUDE_3_7_SONNET_20250219 = ("anthropic", "claude-3-7-sonnet-20250219") CLAUDE_3_5_HAIKU_20241022 = ("anthropic", "claude-3-5-haiku-20241022") diff --git a/notdiamond/llms/request.py b/notdiamond/llms/request.py index 4f2bbad5..31f57328 100644 --- a/notdiamond/llms/request.py +++ b/notdiamond/llms/request.py @@ -119,7 +119,14 @@ def model_select_parse(response_code, response_json, llm_configs): )[0] return best_llm, session_id - error_message = response_json["detail"] + # Handle different error response formats + if "error" in response_json and isinstance(response_json["error"], dict): + error_message = response_json["error"].get("message", str(response_json["error"])) + elif "detail" in response_json: + error_message = response_json["detail"] + else: + error_message = str(response_json) + LOGGER.error(f"API error: {response_code}. {error_message}") return None, "NO-SESSION-ID" diff --git a/notdiamond/settings.py b/notdiamond/settings.py index 791708fb..fa68779b 100644 --- a/notdiamond/settings.py +++ b/notdiamond/settings.py @@ -145,7 +145,6 @@ "models": [ "claude-2.1", "claude-3-opus-20240229", - "claude-3-sonnet-20240229", "claude-3-haiku-20240307", "claude-3-5-haiku-20241022", "claude-3-7-sonnet-latest", @@ -164,7 +163,6 @@ "api_key": ANTHROPIC_API_KEY, "support_tools": [ "claude-3-opus-20240229", - "claude-3-sonnet-20240229", "claude-3-haiku-20240307", "claude-3-5-haiku-20241022", "claude-3-7-sonnet-latest", @@ -187,7 +185,6 @@ "openrouter_identifier": { "claude-2.1": "anthropic/claude-2.1", "claude-3-opus-20240229": "anthropic/claude-3-opus", - "claude-3-sonnet-20240229": "anthropic/claude-3-sonnet", "claude-3-haiku-20240307": "anthropic/claude-3-haiku", "claude-3-5-haiku-20241022": "anthropic/claude-3.5-haiku", "claude-3-7-sonnet-latest": "anthropic/claude-3.7-sonnet", @@ -206,7 +203,6 @@ "price": { "claude-2.1": {"input": 8.0, "output": 24.0}, "claude-3-opus-20240229": {"input": 15.0, "output": 75.0}, - "claude-3-sonnet-20240229": {"input": 3.0, "output": 15.0}, "claude-3-haiku-20240307": {"input": 0.25, "output": 1.25}, "claude-3-5-haiku-20241022": {"input": 1.0, "output": 5.0}, "claude-3-7-sonnet-latest": {"input": 3.0, "output": 15.0}, diff --git a/tests/test_documentation/test_fallback_and_custom.py b/tests/test_documentation/test_fallback_and_custom.py index 6560ece6..36cf9f25 100644 --- a/tests/test_documentation/test_fallback_and_custom.py +++ b/tests/test_documentation/test_fallback_and_custom.py @@ -13,7 +13,6 @@ def test_fallback_model(): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-2.1", - "anthropic/claude-3-sonnet-20240229", "anthropic/claude-3-opus-20240229", "google/gemini-pro", ] @@ -33,7 +32,6 @@ def test_set_max_model_depth(): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-2.1", - "anthropic/claude-3-sonnet-20240229", "anthropic/claude-3-opus-20240229", "google/gemini-pro", ] diff --git a/tests/test_documentation/test_getting_started.py b/tests/test_documentation/test_getting_started.py index 2237ba2d..87b852a0 100644 --- a/tests/test_documentation/test_getting_started.py +++ b/tests/test_documentation/test_getting_started.py @@ -23,7 +23,7 @@ def test_main_example(): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-3-haiku-20240307", - "anthropic/claude-3-sonnet-20240229", + "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-3-opus-20240229", ] @@ -55,7 +55,7 @@ def test_pass_array_of_messages(start_prompt): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-3-haiku-20240307", - "anthropic/claude-3-sonnet-20240229", + "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-3-opus-20240229", ] @@ -87,7 +87,7 @@ def test_programatic_define_llm_configs(start_prompt): ), LLMConfig( provider="anthropic", - model="claude-3-sonnet-20240229", + model="claude-sonnet-4-5-20250929", temperature=0.8, max_tokens=256, ), @@ -118,7 +118,7 @@ def test_model_select(start_prompt): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-3-haiku-20240307", - "anthropic/claude-3-sonnet-20240229", + "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-3-opus-20240229", ] diff --git a/tests/test_documentation/test_openrouter.py b/tests/test_documentation/test_openrouter.py index a5b468b0..e4508bab 100644 --- a/tests/test_documentation/test_openrouter.py +++ b/tests/test_documentation/test_openrouter.py @@ -27,7 +27,7 @@ def test_openrouter_integration(): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-3-haiku-20240307", - "anthropic/claude-3-sonnet-20240229", + "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-3-opus-20240229", ] diff --git a/tests/test_documentation/test_personalization.py b/tests/test_documentation/test_personalization.py index 981795fa..41fea636 100644 --- a/tests/test_documentation/test_personalization.py +++ b/tests/test_documentation/test_personalization.py @@ -19,7 +19,7 @@ def test_personalization(): "openai/gpt-4-1106-preview", "openai/gpt-4-turbo-preview", "anthropic/claude-2.1", - "anthropic/claude-3-sonnet-20240229", + "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-3-opus-20240229", "google/gemini-pro", ] diff --git a/tests/test_llm_calls/cassettes/test_anthropic/Test_Anthropic_LLMs.test_claude_3_sonnet_with_openai_tool_calling.yaml b/tests/test_llm_calls/cassettes/test_anthropic/Test_Anthropic_LLMs.test_claude_3_sonnet_with_openai_tool_calling.yaml deleted file mode 100644 index 34a31d60..00000000 --- a/tests/test_llm_calls/cassettes/test_anthropic/Test_Anthropic_LLMs.test_claude_3_sonnet_with_openai_tool_calling.yaml +++ /dev/null @@ -1,160 +0,0 @@ -interactions: -- request: - body: '{"messages": [{"role": "user", "content": "How much is 3 + 5?"}], "llm_providers": - [{"provider": "anthropic", "model": "claude-3-sonnet-20240229", "is_custom": - false, "context_length": null, "input_price": null, "output_price": null, "latency": - null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": true, "tools": - [{"type": "function", "function": {"name": "add_fct", "description": "Add two - numbers", "parameters": {"type": "object", "properties": {"a": {"type": "integer", - "description": "The first number"}, "b": {"type": "integer", "description": - "The second number"}}, "required": ["a", "b"]}}}]}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - Content-Length: - - '615' - User-Agent: - - Python-SDK/0.3.41 - content-type: - - application/json - method: POST - uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect - response: - body: - string: '{"providers":[{"provider":"anthropic","model":"claude-3-sonnet-20240229"}],"session_id":"e22fcc36-f5ec-40a0-99c0-df2381512d5a"}' - headers: - CF-RAY: - - 925d002b88de1c2f-SOF - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Tue, 25 Mar 2025 08:22:29 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - content-length: - - '127' - rndr-id: - - abadc430-9527-4421 - vary: - - Accept-Encoding - x-render-origin-server: - - uvicorn - status: - code: 200 - message: OK -- request: - body: '{"max_tokens": 200, "messages": [{"role": "user", "content": "How much - is 3 + 5?"}], "model": "claude-3-sonnet-20240229", "tools": [{"name": "add_fct", - "description": "Add two numbers", "input_schema": {"type": "object", "properties": - {"a": {"type": "integer", "description": "The first number"}, "b": {"type": - "integer", "description": "The second number"}}, "required": ["a", "b"]}}]}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2023-06-01' - connection: - - keep-alive - content-length: - - '386' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - Anthropic/Python 0.49.0 - x-api-key: - - REDACTED - x-stainless-arch: - - x64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - Linux - x-stainless-package-version: - - 0.49.0 - x-stainless-read-timeout: - - '120.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.12.3 - x-stainless-timeout: - - NOT_GIVEN - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id":"msg_01K1T4Mi5GZLuRtUinyjveLz","type":"message","role":"assistant","model":"claude-3-sonnet-20240229","content":[{"type":"text","text":"Okay, - let''s use the \"add_fct\" tool to add 3 and 5:"},{"type":"tool_use","id":"toolu_01AuiHVSoidXxUb2BsT5QzWq","name":"add_fct","input":{"a":3,"b":5}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":257,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":95}}' - headers: - CF-RAY: - - 925d00306efab654-VIE - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Tue, 25 Mar 2025 08:22:30 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Robots-Tag: - - none - anthropic-organization-id: - - a2610b2c-742e-455d-9552-6abe105b2a44 - anthropic-ratelimit-input-tokens-limit: - - '400000' - anthropic-ratelimit-input-tokens-remaining: - - '400000' - anthropic-ratelimit-input-tokens-reset: - - '2025-03-25T08:22:29Z' - anthropic-ratelimit-output-tokens-limit: - - '80000' - anthropic-ratelimit-output-tokens-remaining: - - '80000' - anthropic-ratelimit-output-tokens-reset: - - '2025-03-25T08:22:30Z' - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - anthropic-ratelimit-requests-reset: - - '2025-03-25T08:22:29Z' - anthropic-ratelimit-tokens-limit: - - '480000' - anthropic-ratelimit-tokens-remaining: - - '480000' - anthropic-ratelimit-tokens-reset: - - '2025-03-25T08:22:29Z' - cf-cache-status: - - DYNAMIC - content-length: - - '449' - request-id: - - req_016czM8v8AMfmhmY62zL8tXm - via: - - 1.1 google - status: - code: 200 - message: OK -version: 1 diff --git a/tests/test_llm_calls/cassettes/test_anthropic/Test_Anthropic_LLMs.test_claude_3_sonnet_with_tool_calling.yaml b/tests/test_llm_calls/cassettes/test_anthropic/Test_Anthropic_LLMs.test_claude_3_sonnet_with_tool_calling.yaml deleted file mode 100644 index f37795ae..00000000 --- a/tests/test_llm_calls/cassettes/test_anthropic/Test_Anthropic_LLMs.test_claude_3_sonnet_with_tool_calling.yaml +++ /dev/null @@ -1,158 +0,0 @@ -interactions: -- request: - body: '{"messages": [{"role": "user", "content": "How much is 3 + 5?"}], "llm_providers": - [{"provider": "anthropic", "model": "claude-3-sonnet-20240229", "is_custom": - false, "context_length": null, "input_price": null, "output_price": null, "latency": - null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": true, "tools": - [{"type": "function", "function": {"name": "add_fct", "description": "Adds a - and b.", "parameters": {"type": "object", "properties": {"a": {"type": "integer"}, - "b": {"type": "integer"}}, "required": ["a", "b"]}}}]}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - Content-Length: - - '542' - User-Agent: - - Python-SDK/0.3.41 - content-type: - - application/json - method: POST - uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect - response: - body: - string: '{"providers":[{"provider":"anthropic","model":"claude-3-sonnet-20240229"}],"session_id":"665d9ae0-4838-42a7-8c5e-66b62b24c968"}' - headers: - CF-RAY: - - 925d001c3e198eec-SOF - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Tue, 25 Mar 2025 08:22:26 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - content-length: - - '127' - rndr-id: - - 0c8a3881-4473-47fd - vary: - - Accept-Encoding - x-render-origin-server: - - uvicorn - status: - code: 200 - message: OK -- request: - body: '{"max_tokens": 200, "messages": [{"role": "user", "content": "How much - is 3 + 5?"}], "model": "claude-3-sonnet-20240229", "tools": [{"name": "add_fct", - "description": "Adds a and b.", "input_schema": {"type": "object", "properties": - {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"]}}]}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2023-06-01' - connection: - - keep-alive - content-length: - - '313' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - Anthropic/Python 0.49.0 - x-api-key: - - REDACTED - x-stainless-arch: - - x64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - Linux - x-stainless-package-version: - - 0.49.0 - x-stainless-read-timeout: - - '120.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.12.3 - x-stainless-timeout: - - NOT_GIVEN - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"id":"msg_01QoMZsMN28EwN3jfhupfA1n","type":"message","role":"assistant","model":"claude-3-sonnet-20240229","content":[{"type":"text","text":"Okay, - let''s use the available tool to add these numbers:"},{"type":"tool_use","id":"toolu_01Mo4VwkaVAZFYk4tKnZ2Xao","name":"add_fct","input":{"a":3,"b":5}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":243,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":86}}' - headers: - CF-RAY: - - 925d002108263a0d-VIE - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Tue, 25 Mar 2025 08:22:28 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Robots-Tag: - - none - anthropic-organization-id: - - a2610b2c-742e-455d-9552-6abe105b2a44 - anthropic-ratelimit-input-tokens-limit: - - '400000' - anthropic-ratelimit-input-tokens-remaining: - - '400000' - anthropic-ratelimit-input-tokens-reset: - - '2025-03-25T08:22:27Z' - anthropic-ratelimit-output-tokens-limit: - - '80000' - anthropic-ratelimit-output-tokens-remaining: - - '80000' - anthropic-ratelimit-output-tokens-reset: - - '2025-03-25T08:22:28Z' - anthropic-ratelimit-requests-limit: - - '4000' - anthropic-ratelimit-requests-remaining: - - '3999' - anthropic-ratelimit-requests-reset: - - '2025-03-25T08:22:26Z' - anthropic-ratelimit-tokens-limit: - - '480000' - anthropic-ratelimit-tokens-remaining: - - '480000' - anthropic-ratelimit-tokens-reset: - - '2025-03-25T08:22:27Z' - cf-cache-status: - - DYNAMIC - content-length: - - '453' - request-id: - - req_01JHq913Tmt9AHshjMKACPCe - via: - - 1.1 google - status: - code: 200 - message: OK -version: 1 diff --git a/tests/test_llm_calls/test_anthropic.py b/tests/test_llm_calls/test_anthropic.py index cfd5031c..d37b0585 100644 --- a/tests/test_llm_calls/test_anthropic.py +++ b/tests/test_llm_calls/test_anthropic.py @@ -84,36 +84,6 @@ def test_claude_3_opus_response_model(self, response_model): assert result.setup assert result.punchline - def test_claude_3_sonnet_with_tool_calling(self, tools_fixture): - provider = NDLLMProviders.CLAUDE_3_SONNET_20240229 - provider.kwargs = {"max_tokens": 200} - nd_llm = NotDiamond( - llm_configs=[provider], latency_tracking=False, hash_content=True - ) - nd_llm = nd_llm.bind_tools(tools_fixture) - result, session_id, _ = nd_llm.invoke( - [{"role": "user", "content": "How much is 3 + 5?"}] - ) - - assert len(result.tool_calls) == 1 - assert result.tool_calls[0]["name"] == "add_fct" - - def test_claude_3_sonnet_with_openai_tool_calling( - self, openai_tools_fixture - ): - provider = NDLLMProviders.CLAUDE_3_SONNET_20240229 - provider.kwargs = {"max_tokens": 200} - nd_llm = NotDiamond( - llm_configs=[provider], latency_tracking=False, hash_content=True - ) - nd_llm = nd_llm.bind_tools(openai_tools_fixture) - result, session_id, _ = nd_llm.invoke( - [{"role": "user", "content": "How much is 3 + 5?"}] - ) - - assert len(result.tool_calls) == 1 - assert result.tool_calls[0]["name"] == "add_fct" - def test_claude_3_haiku_with_tool_calling(self, tools_fixture): provider = NDLLMProviders.CLAUDE_3_HAIKU_20240307 provider.kwargs = {"max_tokens": 200} diff --git a/tests/test_toolkit/langchain/test_integration.py b/tests/test_toolkit/langchain/test_integration.py index 42db63a1..fa7c01a6 100644 --- a/tests/test_toolkit/langchain/test_integration.py +++ b/tests/test_toolkit/langchain/test_integration.py @@ -70,7 +70,7 @@ def fn(x: str) -> str: "langchain_anthropic.ChatAnthropic", ), ( - "anthropic/claude-3-sonnet-20240229", + "anthropic/claude-sonnet-4-5-20250929", "langchain_anthropic.ChatAnthropic", ), (