Skip to content
This repository was archived by the owner on Dec 11, 2025. It is now read-only.

Update model support and tests #96

Closed
acompa wants to merge 1 commit into from
Closed

Update model support and tests #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions notdiamond/llms/providers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class NDLLMProviders(Enum):
GPT_4_TURBO_2024_04_09 (NDLLMProvider): refers to 'gpt-4-turbo-2024-04-09' model by OpenAI
GPT_4o_2024_05_13 (NDLLMProvider): refers to 'gpt-4o-2024-05-13' model by OpenAI
GPT_4o_2024_08_06 (NDLLMProvider): refers to 'gpt-4o-2024-08-06' model by OpenAI
GPT_4o_2024_11_20 (NDLLMProvider): refers to 'gpt-4o-2024-11-20' model by OpenAI
GPT_4o (NDLLMProvider): refers to 'gpt-4o' model by OpenAI
GPT_4o_MINI_2024_07_18 (NDLLMProvider): refers to 'gpt-4o-mini-2024-07-18' model by OpenAI
GPT_4o_MINI (NDLLMProvider): refers to 'gpt-4o-mini' model by OpenAI
Expand All @@ -35,9 +36,7 @@ class NDLLMProviders(Enum):
O1_MINI_2024_09_12 (NDLLMProvider): refers to 'o1-mini-2024-09-12' model by OpenAI

CLAUDE_2_1 (NDLLMProvider): refers to 'claude-2.1' model by Anthropic
CLAUDE_3_OPUS_20240229 (NDLLMProvider): refers to 'claude-3-opus-20240229' model by Anthropic
CLAUDE_3_SONNET_20240229 (NDLLMProvider): refers to 'claude-3-sonnet-20240229' model by Anthropic
CLAUDE_3_5_SONNET_20240620 (NDLLMProvider): refers to 'claude-3-5-sonnet-20240620' model by Anthropic
CLAUDE_3_7_SONNET_LATEST (NDLLMProvider): refers to 'claude-3-7-sonnet-latest' model by Anthropic
CLAUDE_3_7_SONNET_20250219 (NDLLMProvider): refers to 'claude-3-7-sonnet-20250219' model by Anthropic
CLAUDE_3_5_HAIKU_20241022 (NDLLMProvider): refers to 'claude-3-5-haiku-20241022' model by Anthropic
Expand Down Expand Up @@ -107,6 +106,7 @@ class NDLLMProviders(Enum):
GPT_4_TURBO_2024_04_09 = ("openai", "gpt-4-turbo-2024-04-09")
GPT_4o_2024_05_13 = ("openai", "gpt-4o-2024-05-13")
GPT_4o_2024_08_06 = ("openai", "gpt-4o-2024-08-06")
GPT_4o_2024_11_20 = ("openai", "gpt-4o-2024-11-20")
GPT_4o = ("openai", "gpt-4o")
GPT_4o_MINI_2024_07_18 = ("openai", "gpt-4o-mini-2024-07-18")
GPT_4o_MINI = ("openai", "gpt-4o-mini")
Expand All @@ -124,11 +124,7 @@ class NDLLMProviders(Enum):
CHATGPT_4o_LATEST = ("openai", "chatgpt-4o-latest")

CLAUDE_2_1 = ("anthropic", "claude-2.1")
CLAUDE_3_OPUS_20240229 = ("anthropic", "claude-3-opus-20240229")
CLAUDE_3_SONNET_20240229 = ("anthropic", "claude-3-sonnet-20240229")
CLAUDE_3_5_SONNET_20240620 = ("anthropic", "claude-3-5-sonnet-20240620")
CLAUDE_3_5_SONNET_20241022 = ("anthropic", "claude-3-5-sonnet-20241022")
CLAUDE_3_5_SONNET_LATEST = ("anthropic", "claude-3-5-sonnet-latest")
CLAUDE_3_7_SONNET_LATEST = ("anthropic", "claude-3-7-sonnet-latest")
CLAUDE_3_7_SONNET_20250219 = ("anthropic", "claude-3-7-sonnet-20250219")
CLAUDE_3_5_HAIKU_20241022 = ("anthropic", "claude-3-5-haiku-20241022")
Expand Down
21 changes: 5 additions & 16 deletions notdiamond/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"gpt-4o",
"gpt-4o-2024-05-13",
"gpt-4o-2024-08-06",
"gpt-4o-2024-11-20",
"gpt-4o-mini",
"gpt-4o-mini-2024-07-18",
"gpt-4-turbo-preview",
Expand Down Expand Up @@ -62,6 +63,7 @@
"gpt-4o",
"gpt-4o-2024-05-13",
"gpt-4o-2024-08-06",
"gpt-4o-2024-11-20",
"gpt-4o-mini",
"gpt-4o-mini-2024-07-18",
"gpt-4-turbo-preview",
Expand All @@ -84,6 +86,7 @@
"gpt-4o",
"gpt-4o-2024-05-13",
"gpt-4o-2024-08-06",
"gpt-4o-2024-11-20",
"gpt-4o-mini",
"gpt-4o-mini-2024-07-18",
"gpt-4-turbo-preview",
Expand All @@ -108,6 +111,7 @@
"gpt-4o": "openai/gpt-4o",
"gpt-4o-2024-05-13": "openai/gpt-4o-2024-05-13",
"gpt-4o-2024-08-06": "openai/gpt-4o-2024-08-06",
"gpt-4o-2024-11-20": "openai/gpt-4o-2024-11-20",
"gpt-4o-mini": "openai/gpt-4o-mini",
"gpt-4o-mini-2024-07-18": "openai/gpt-4o-mini-2024-07-18",
"gpt-4-turbo-preview": "openai/gpt-4-turbo-preview",
Expand All @@ -134,6 +138,7 @@
"gpt-4o": {"input": 5.0, "output": 15.0},
"gpt-4o-2024-05-13": {"input": 5.0, "output": 15.0},
"gpt-4o-2024-08-06": {"input": 2.5, "output": 10.0},
"gpt-4o-2024-11-20": {"input": 2.5, "output": 10.0},
"gpt-4o-mini": {"input": 0.15, "output": 0.6},
"gpt-4o-mini-2024-07-18": {"input": 0.15, "output": 0.6},
"gpt-4-turbo-preview": {"input": 10.0, "output": 30.0},
Expand All @@ -155,13 +160,9 @@
"anthropic": {
"models": [
"claude-2.1",
"claude-3-opus-20240229",
"claude-3-sonnet-20240229",
"claude-3-haiku-20240307",
"claude-3-5-haiku-20241022",
"claude-3-5-sonnet-20240620",
"claude-3-5-sonnet-20241022",
"claude-3-5-sonnet-latest",
"claude-3-7-sonnet-latest",
"claude-3-7-sonnet-20250219",
"claude-opus-4-20250514",
Expand All @@ -173,13 +174,9 @@
],
"api_key": ANTHROPIC_API_KEY,
"support_tools": [
"claude-3-opus-20240229",
"claude-3-sonnet-20240229",
"claude-3-haiku-20240307",
"claude-3-5-sonnet-20240620",
"claude-3-5-sonnet-20241022",
"claude-3-5-haiku-20241022",
"claude-3-5-sonnet-latest",
"claude-3-7-sonnet-latest",
"claude-3-7-sonnet-20250219",
"claude-opus-4-20250514",
Expand All @@ -191,15 +188,11 @@
],
"support_response_model": [
"claude-2.1",
"claude-3-opus-20240229",
],
"openrouter_identifier": {
"claude-2.1": "anthropic/claude-2.1",
"claude-3-opus-20240229": "anthropic/claude-3-opus",
"claude-3-sonnet-20240229": "anthropic/claude-3-sonnet",
"claude-3-haiku-20240307": "anthropic/claude-3-haiku",
"claude-3-5-sonnet-20240620": "anthropic/claude-3.5-sonnet-20240620",
"claude-3-5-sonnet-latest": "anthropic/claude-3.5-sonnet",
"claude-3-5-haiku-20241022": "anthropic/claude-3.5-haiku",
"claude-3-7-sonnet-latest": "anthropic/claude-3.7-sonnet",
"claude-3-7-sonnet-20250219": "anthropic/claude-3.7-sonnet",
Expand All @@ -212,13 +205,9 @@
},
"price": {
"claude-2.1": {"input": 8.0, "output": 24.0},
"claude-3-opus-20240229": {"input": 15.0, "output": 75.0},
"claude-3-sonnet-20240229": {"input": 3.0, "output": 15.0},
"claude-3-haiku-20240307": {"input": 0.25, "output": 1.25},
"claude-3-5-haiku-20241022": {"input": 1.0, "output": 5.0},
"claude-3-5-sonnet-20240620": {"input": 3.0, "output": 15.0},
"claude-3-5-sonnet-20241022": {"input": 3.0, "output": 15.0},
"claude-3-5-sonnet-latest": {"input": 3.0, "output": 15.0},
"claude-3-7-sonnet-latest": {"input": 3.0, "output": 15.0},
"claude-3-7-sonnet-20250219": {"input": 3.0, "output": 15.0},
"claude-opus-4-20250514": {"input": 15.0, "output": 75.0},
Expand Down
105 changes: 2 additions & 103 deletions tests/test_llm_calls/test_anthropic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,50 +39,7 @@ def test_claude_21_response_model(self, response_model):
assert result.setup
assert result.punchline

def test_claude_3_opus_with_tool_calling(self, tools_fixture):
provider = NDLLMProviders.CLAUDE_3_OPUS_20240229
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
nd_llm = nd_llm.bind_tools(tools_fixture)
result, session_id, _ = nd_llm.invoke(
[{"role": "user", "content": "How much is 3 + 5?"}]
)

assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"

def test_claude_3_opus_with_openai_tool_calling(
self, openai_tools_fixture
):
provider = NDLLMProviders.CLAUDE_3_OPUS_20240229
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
nd_llm = nd_llm.bind_tools(openai_tools_fixture)
result, session_id, _ = nd_llm.invoke(
[{"role": "user", "content": "How much is 3 + 5?"}]
)

assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"

def test_claude_3_opus_response_model(self, response_model):
provider = NDLLMProviders.CLAUDE_3_OPUS_20240229
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
result, _, _ = nd_llm.invoke(
[{"role": "user", "content": "Tell me a joke"}],
response_model=response_model,
)

assert isinstance(result, response_model)
assert result.setup
assert result.punchline
# Removed tests for deprecated Claude 3 Opus models

def test_claude_3_sonnet_with_tool_calling(self, tools_fixture):
provider = NDLLMProviders.CLAUDE_3_SONNET_20240229
Expand Down Expand Up @@ -128,65 +85,7 @@ def test_claude_3_haiku_with_tool_calling(self, tools_fixture):
assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"

def test_claude_3_5_sonnet_with_tool_calling(self, tools_fixture):
provider = NDLLMProviders.CLAUDE_3_5_SONNET_20240620
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
nd_llm = nd_llm.bind_tools(tools_fixture)
result, session_id, _ = nd_llm.invoke(
[{"role": "user", "content": "How much is 3 + 5?"}]
)

assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"

def test_claude_3_5_sonnet_with_openai_tool_calling(
self, openai_tools_fixture
):
provider = NDLLMProviders.CLAUDE_3_5_SONNET_20240620
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
nd_llm = nd_llm.bind_tools(openai_tools_fixture)
result, session_id, _ = nd_llm.invoke(
[{"role": "user", "content": "How much is 3 + 5?"}]
)

assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"

def test_claude_3_5_sonnet_20241022_with_tool_calling(self, tools_fixture):
provider = NDLLMProviders.CLAUDE_3_5_SONNET_20241022
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
nd_llm = nd_llm.bind_tools(tools_fixture)
result, session_id, _ = nd_llm.invoke(
[{"role": "user", "content": "How much is 3 + 5?"}]
)

assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"

def test_claude_3_5_sonnet_20241022_with_openai_tool_calling(
self, openai_tools_fixture
):
provider = NDLLMProviders.CLAUDE_3_5_SONNET_20241022
provider.kwargs = {"max_tokens": 200}
nd_llm = NotDiamond(
llm_configs=[provider], latency_tracking=False, hash_content=True
)
nd_llm = nd_llm.bind_tools(openai_tools_fixture)
result, session_id, _ = nd_llm.invoke(
[{"role": "user", "content": "How much is 3 + 5?"}]
)

assert len(result.tool_calls) == 1
assert result.tool_calls[0]["name"] == "add_fct"
# Removed tests for deprecated Claude 3.5 Sonnet models

def test_claude_3_5_haiku_with_tool_calling(self, tools_fixture):
provider = NDLLMProviders.CLAUDE_3_5_HAIKU_20241022
Expand Down
3 changes: 2 additions & 1 deletion tests/test_toolkit/rag/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ def gpt_4o():

@pytest.fixture
def sonnet_3_5():
return LLMConfig.from_string("anthropic/claude-3-5-sonnet-20241022")
# Deprecated; replacing with supported claude-3-5-haiku-20241022
return LLMConfig.from_string("anthropic/claude-3-5-haiku-20241022")


@pytest.fixture
Expand Down
Loading