Skip to content

Commit f06bc2e

Browse files
mathisluckamathislucka
authored
feat: add additional test cases (#99)
1 parent fe009e7 commit f06bc2e

4 files changed

Lines changed: 304 additions & 0 deletions

File tree

‎src/deepset_mcp/benchmark/tasks/jinja_str_instead_of_complex_type.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
name: "jinja_str_instead_of_complex_type"
2+
objective: "Add parentheses after the filter in the OutputAdapter to get correct attribute access."
3+
prompt: "Can you check why my chat agent pipeline doesn't work."
4+
query_yaml: "pipelines/chat_agent_jinja_syntax.yml"
5+
query_name: "chat-agent"
6+
tags:
7+
- "debug"
8+
- "jinja"
9+
judge_prompt: |
10+
The Agent has:
11+
- added parentheses around messages|last so that attribute access works correctly

‎src/deepset_mcp/benchmark/tasks/jinja_syntax_error.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
name: "jinja_syntax_error"
2+
objective: "Add double braces to the OutputAdapter to fix the syntax error."
3+
prompt: "Can you check why my chat agent pipeline doesn't work."
4+
query_yaml: "pipelines/chat_agent_jinja_str.yml"
5+
query_name: "chat-agent"
6+
tags:
7+
- "debug"
8+
- "jinja"
9+
judge_prompt: |
10+
The Agent has:
11+
- successfully fixed the template on the OutputAdapter to not contain additional whitespace

‎src/deepset_mcp/benchmark/tasks/pipelines/chat_agent_jinja_str.yml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# If you need help with the YAML format, have a look at https://docs.cloud.deepset.ai/v2.0/docs/create-a-pipeline#create-a-pipeline-using-pipeline-editor.
2+
# This section defines components that you want to use in your pipelines. Each component must have a name and a type. You can also set the component's parameters here.
3+
# The name is up to you, you can give your component a friendly name. You then use components' names when specifying the connections in the pipeline.
4+
# Type is the class path of the component. You can check the type on the component's documentation page.
5+
components:
6+
adapter:
7+
init_parameters:
8+
custom_filters: {}
9+
output_type: List[str]
10+
template: ' {{ [(messages|last).text] }} '
11+
unsafe: false
12+
type: haystack.components.converters.output_adapter.OutputAdapter
13+
agent:
14+
init_parameters:
15+
chat_generator:
16+
init_parameters:
17+
api_base_url:
18+
api_key:
19+
env_vars:
20+
- OPENAI_API_KEY
21+
strict: false
22+
type: env_var
23+
generation_kwargs: {}
24+
max_retries:
25+
model: gpt-4o
26+
organization:
27+
streaming_callback:
28+
timeout:
29+
tools:
30+
tools_strict: false
31+
type: haystack.components.generators.chat.openai.OpenAIChatGenerator
32+
exit_conditions:
33+
- text
34+
max_agent_steps: 100
35+
raise_on_tool_invocation_failure: false
36+
state_schema: {}
37+
streaming_callback:
38+
system_prompt: |-
39+
You are a deep research assistant.
40+
You create comprehensive research reports to answer the user's questions.
41+
You use the 'search'-tool to answer any questions.
42+
You perform multiple searches until you have the information you need to answer the question.
43+
Make sure you research different aspects of the question.
44+
Use markdown to format your response.
45+
When you use information from the websearch results, cite your sources using markdown links.
46+
It is important that you cite accurately.
47+
tools:
48+
- data:
49+
component:
50+
init_parameters:
51+
input_mapping:
52+
query:
53+
- search.query
54+
output_mapping:
55+
builder.prompt: result
56+
pipeline:
57+
components:
58+
builder:
59+
init_parameters:
60+
required_variables: "*"
61+
template: |-
62+
{% for doc in docs %}
63+
{% if doc.content and doc.meta.url|length > 0 %}
64+
<search-result url="{{ doc.meta.url }}">
65+
{{ doc.content|truncate(25000) }}
66+
</search-result>
67+
{% endif %}
68+
{% endfor %}
69+
variables:
70+
type: haystack.components.builders.prompt_builder.PromptBuilder
71+
converter:
72+
init_parameters:
73+
extraction_kwargs: {}
74+
store_full_path: false
75+
type: haystack.components.converters.html.HTMLToDocument
76+
fetcher:
77+
init_parameters:
78+
raise_on_failure: false
79+
retry_attempts: 2
80+
timeout: 3
81+
user_agents:
82+
- haystack/LinkContentFetcher/2.11.1
83+
type: haystack.components.fetchers.link_content.LinkContentFetcher
84+
search:
85+
init_parameters:
86+
api_key:
87+
env_vars:
88+
- SERPERDEV_API_KEY
89+
strict: false
90+
type: env_var
91+
search_params: {}
92+
top_k: 10
93+
type: haystack.components.websearch.serper_dev.SerperDevWebSearch
94+
connection_type_validation: true
95+
connections:
96+
- receiver: fetcher.urls
97+
sender: search.links
98+
- receiver: converter.sources
99+
sender: fetcher.streams
100+
- receiver: builder.docs
101+
sender: converter.documents
102+
max_runs_per_component: 100
103+
metadata: {}
104+
type: haystack.core.super_component.super_component.SuperComponent
105+
description: Use this tool to search for information on the internet.
106+
inputs_from_state:
107+
name: search
108+
parameters:
109+
type: haystack.tools.component_tool.ComponentTool
110+
type: haystack.components.agents.agent.Agent
111+
answer_builder:
112+
init_parameters:
113+
pattern:
114+
reference_pattern:
115+
type: haystack.components.builders.answer_builder.AnswerBuilder
116+
history_parser:
117+
init_parameters: {}
118+
type: dc_custom_component.components.parsers.chat_history_parser.DeepsetChatHistoryParser
119+
120+
121+
connections:
122+
- receiver: agent.messages
123+
sender: history_parser.messages
124+
- receiver: adapter.messages
125+
sender: agent.messages
126+
- receiver: answer_builder.replies
127+
sender: adapter.output
128+
129+
inputs:
130+
query:
131+
- answer_builder.query
132+
- history_parser.history_and_query
133+
134+
outputs:
135+
answers: answer_builder.answers
136+
137+
pipeline_output_type: chat
138+
139+
max_runs_per_component: 100
140+
141+
metadata: {}

‎src/deepset_mcp/benchmark/tasks/pipelines/chat_agent_jinja_syntax.yml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# If you need help with the YAML format, have a look at https://docs.cloud.deepset.ai/v2.0/docs/create-a-pipeline#create-a-pipeline-using-pipeline-editor.
2+
# This section defines components that you want to use in your pipelines. Each component must have a name and a type. You can also set the component's parameters here.
3+
# The name is up to you, you can give your component a friendly name. You then use components' names when specifying the connections in the pipeline.
4+
# Type is the class path of the component. You can check the type on the component's documentation page.
5+
components:
6+
adapter:
7+
init_parameters:
8+
custom_filters: {}
9+
output_type: List[str]
10+
template: '{{ [messages|last.text] }}'
11+
unsafe: false
12+
type: haystack.components.converters.output_adapter.OutputAdapter
13+
agent:
14+
init_parameters:
15+
chat_generator:
16+
init_parameters:
17+
api_base_url:
18+
api_key:
19+
env_vars:
20+
- OPENAI_API_KEY
21+
strict: false
22+
type: env_var
23+
generation_kwargs: {}
24+
max_retries:
25+
model: gpt-4o
26+
organization:
27+
streaming_callback:
28+
timeout:
29+
tools:
30+
tools_strict: false
31+
type: haystack.components.generators.chat.openai.OpenAIChatGenerator
32+
exit_conditions:
33+
- text
34+
max_agent_steps: 100
35+
raise_on_tool_invocation_failure: false
36+
state_schema: {}
37+
streaming_callback:
38+
system_prompt: |-
39+
You are a deep research assistant.
40+
You create comprehensive research reports to answer the user's questions.
41+
You use the 'search'-tool to answer any questions.
42+
You perform multiple searches until you have the information you need to answer the question.
43+
Make sure you research different aspects of the question.
44+
Use markdown to format your response.
45+
When you use information from the websearch results, cite your sources using markdown links.
46+
It is important that you cite accurately.
47+
tools:
48+
- data:
49+
component:
50+
init_parameters:
51+
input_mapping:
52+
query:
53+
- search.query
54+
output_mapping:
55+
builder.prompt: result
56+
pipeline:
57+
components:
58+
builder:
59+
init_parameters:
60+
required_variables: "*"
61+
template: |-
62+
{% for doc in docs %}
63+
{% if doc.content and doc.meta.url|length > 0 %}
64+
<search-result url="{{ doc.meta.url }}">
65+
{{ doc.content|truncate(25000) }}
66+
</search-result>
67+
{% endif %}
68+
{% endfor %}
69+
variables:
70+
type: haystack.components.builders.prompt_builder.PromptBuilder
71+
converter:
72+
init_parameters:
73+
extraction_kwargs: {}
74+
store_full_path: false
75+
type: haystack.components.converters.html.HTMLToDocument
76+
fetcher:
77+
init_parameters:
78+
raise_on_failure: false
79+
retry_attempts: 2
80+
timeout: 3
81+
user_agents:
82+
- haystack/LinkContentFetcher/2.11.1
83+
type: haystack.components.fetchers.link_content.LinkContentFetcher
84+
search:
85+
init_parameters:
86+
api_key:
87+
env_vars:
88+
- SERPERDEV_API_KEY
89+
strict: false
90+
type: env_var
91+
search_params: {}
92+
top_k: 10
93+
type: haystack.components.websearch.serper_dev.SerperDevWebSearch
94+
connection_type_validation: true
95+
connections:
96+
- receiver: fetcher.urls
97+
sender: search.links
98+
- receiver: converter.sources
99+
sender: fetcher.streams
100+
- receiver: builder.docs
101+
sender: converter.documents
102+
max_runs_per_component: 100
103+
metadata: {}
104+
type: haystack.core.super_component.super_component.SuperComponent
105+
description: Use this tool to search for information on the internet.
106+
inputs_from_state:
107+
name: search
108+
parameters:
109+
type: haystack.tools.component_tool.ComponentTool
110+
type: haystack.components.agents.agent.Agent
111+
answer_builder:
112+
init_parameters:
113+
pattern:
114+
reference_pattern:
115+
type: haystack.components.builders.answer_builder.AnswerBuilder
116+
history_parser:
117+
init_parameters: {}
118+
type: dc_custom_component.components.parsers.chat_history_parser.DeepsetChatHistoryParser
119+
120+
121+
connections:
122+
- receiver: agent.messages
123+
sender: history_parser.messages
124+
- receiver: adapter.messages
125+
sender: agent.messages
126+
- receiver: answer_builder.replies
127+
sender: adapter.output
128+
129+
inputs:
130+
query:
131+
- answer_builder.query
132+
- history_parser.history_and_query
133+
134+
outputs:
135+
answers: answer_builder.answers
136+
137+
pipeline_output_type: chat
138+
139+
max_runs_per_component: 100
140+
141+
metadata: {}

0 commit comments

Comments
 (0)