Skip to content

Commit 13df369

Browse files
giuliastfgiuliastf
committed
feat(testcases): Integration test for simple hitl agent
1 parent a3becbc commit 13df369

13 files changed

Lines changed: 3151 additions & 3 deletions

File tree

‎.github/workflows/integration_tests.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ jobs:
1717
matrix:
1818
include:
1919
- build-dir: quickstart-agent
20+
- build-dir: simple-hitl-agent
2021

2122
steps:
2223
- name: Checkout code

‎testcases/quickstart-agent/src/assert.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import sys
33
import json
44

5-
print("Checking company research agent output...")
5+
print("Checking quickstart agent output...")
66

77
# Check NuGet package
88
uipath_dir = ".uipath"
@@ -38,7 +38,7 @@
3838

3939
print("Agent execution status: successful")
4040

41-
# Check required fields for company research agent
41+
# Check required fields for quickstart agent
4242
if "output" not in output_data:
4343
print("Missing 'output' field in agent response")
4444
sys.exit(1)
@@ -66,7 +66,7 @@
6666
print(f"Critique: {critique}")
6767

6868
print("Required fields validation passed")
69-
print("Company research agent working correctly.")
69+
print("Quickstart agent working correctly.")
7070

7171
except Exception as e:
7272
print(f"Error checking output: {e}")

‎testcases/simple-hitl-agent/.env.example‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
OPENAI_API_KEY=xxx

‎testcases/simple-hitl-agent/Dockerfile‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
FROM ghcr.io/astral-sh/uv:python3.12-bookworm
2+
3+
WORKDIR /app
4+
5+
COPY . .
6+
7+
WORKDIR /app/testcases/simple-hitl-agent
8+
9+
RUN uv sync
10+
11+
ARG CLIENT_ID
12+
ARG CLIENT_SECRET
13+
ARG BASE_URL
14+
15+
RUN if [ -z "$CLIENT_ID" ]; then echo "CLIENT_ID build arg is required" && exit 1; fi
16+
RUN if [ -z "$CLIENT_SECRET" ]; then echo "CLIENT_SECRET build arg is required" && exit 1; fi
17+
RUN if [ -z "$BASE_URL" ]; then echo "BASE_URL build arg is required" && exit 1; fi
18+
19+
# Set environment variables for runtime
20+
ENV CLIENT_ID=$CLIENT_ID
21+
ENV CLIENT_SECRET=$CLIENT_SECRET
22+
ENV BASE_URL=$BASE_URL
23+
ENV TAVILY_API_KEY=${TAVILY_API_KEY:-""}
24+
ENV UIPATH_TENANT_ID=${UIPATH_TENANT_ID:-""}
25+
ENV UIPATH_JOB_KEY=8c6a342e-036e-492c-a3d2-99e66f6554ce
26+
27+
# Authenticate with UiPath during build
28+
RUN uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL"
29+
30+
RUN uv run uipath pack
31+
32+
# Run the agent with input from input.json
33+
RUN AGENT_INPUT=$(cat input.json) && uv run uipath run agent "$AGENT_INPUT"
34+
RUN HUMAN_RESPONSE=$(cat human_response.json) && uv run uipath run agent "$HUMAN_RESPONSE" --resume
35+
36+
# Run the Python assert script to validate output
37+
RUN python src/assert.py

‎testcases/simple-hitl-agent/agent.mermaid‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
flowchart TD
2+
step__done["_done"]:::stepStyle
3+
step_aggregate_tool_results["aggregate_tool_results"]:::stepStyle
4+
step_call_tool["call_tool"]:::stepStyle
5+
step_init_run["init_run"]:::stepStyle
6+
step_parse_agent_output["parse_agent_output"]:::stepStyle
7+
step_run_agent_step["run_agent_step"]:::stepStyle
8+
step_setup_agent["setup_agent"]:::stepStyle
9+
event_StopEvent([<p>StopEvent</p>]):::stopEventStyle
10+
event_ToolCallResult([<p>ToolCallResult</p>]):::defaultEventStyle
11+
event_AgentInput([<p>AgentInput</p>]):::defaultEventStyle
12+
event_ToolCall([<p>ToolCall</p>]):::defaultEventStyle
13+
event_AgentWorkflowStartEvent([<p>AgentWorkflowStartEvent</p>]):::defaultEventStyle
14+
event_AgentOutput([<p>AgentOutput</p>]):::defaultEventStyle
15+
event_AgentSetup([<p>AgentSetup</p>]):::defaultEventStyle
16+
event_StopEvent --> step__done
17+
step_aggregate_tool_results --> event_AgentInput
18+
step_aggregate_tool_results --> event_StopEvent
19+
event_ToolCallResult --> step_aggregate_tool_results
20+
step_call_tool --> event_ToolCallResult
21+
event_ToolCall --> step_call_tool
22+
step_init_run --> event_AgentInput
23+
event_AgentWorkflowStartEvent --> step_init_run
24+
step_parse_agent_output --> event_StopEvent
25+
step_parse_agent_output --> event_ToolCall
26+
event_AgentOutput --> step_parse_agent_output
27+
step_run_agent_step --> event_AgentOutput
28+
event_AgentSetup --> step_run_agent_step
29+
step_setup_agent --> event_AgentSetup
30+
event_AgentInput --> step_setup_agent
31+
classDef stepStyle fill:#f2f0ff,line-height:1.2
32+
classDef externalStyle fill:#f2f0ff,line-height:1.2
33+
classDef defaultEventStyle fill-opacity:0
34+
classDef stopEventStyle fill:#bfb6fc
35+
classDef inputRequiredStyle fill:#f2f0ff,line-height:1.2

‎testcases/simple-hitl-agent/human_response.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{ "response": "yes"}

‎testcases/simple-hitl-agent/input.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"user_msg": "research UiPath company"}

‎testcases/simple-hitl-agent/llama_index.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"dependencies": ["."],
3+
"workflows": {
4+
"agent": "./src/main.py:workflow"
5+
},
6+
"env": ".env"
7+
}

‎testcases/simple-hitl-agent/pyproject.toml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[project]
2+
name = "llama-simple-hitl-agent"
3+
version = "0.0.1"
4+
description = "UiPath LlamaIndex Simple HITL Agent"
5+
authors = [{ name = "John Doe", email = "john.doe@myemail.com" }]
6+
readme = { file = "README.md", content-type = "text/markdown" }
7+
requires-python = ">=3.10"
8+
dependencies = [
9+
"uipath-llamaindex",
10+
"llama-index-llms-openai>=0.2.2",
11+
"uipath>=2.0.82",
12+
]
13+
14+
[tool.uv.sources]
15+
uipath-llamaindex = { path = "../../", editable = true }

‎testcases/simple-hitl-agent/src/assert.py‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import os
2+
import sys
3+
import json
4+
5+
print("Checking simple HITL agent output...")
6+
7+
# Check NuGet package
8+
uipath_dir = ".uipath"
9+
if not os.path.exists(uipath_dir):
10+
print("NuGet package directory (.uipath) not found")
11+
sys.exit(1)
12+
13+
nupkg_files = [f for f in os.listdir(uipath_dir) if f.endswith('.nupkg')]
14+
if not nupkg_files:
15+
print("NuGet package file (.nupkg) not found in .uipath directory")
16+
sys.exit(1)
17+
18+
print(f"NuGet package found: {nupkg_files[0]}")
19+
20+
# Check agent output file
21+
output_file = "__uipath/output.json"
22+
if not os.path.isfile(output_file):
23+
print("Agent output file not found")
24+
sys.exit(1)
25+
26+
print("Agent output file found")
27+
28+
# Check status and required fields
29+
try:
30+
with open(output_file, 'r', encoding='utf-8') as f:
31+
output_data = json.load(f)
32+
33+
# Check status
34+
status = output_data.get("status")
35+
if status != "successful":
36+
print(f"Agent execution failed with status: {status}")
37+
sys.exit(1)
38+
39+
print("Agent execution status: successful")
40+
41+
# Check required fields for simple HITL agent
42+
if "output" not in output_data:
43+
print("Missing 'output' field in agent response")
44+
sys.exit(1)
45+
46+
output_content = output_data["output"]
47+
48+
# Check for agent response
49+
if "response" not in output_content:
50+
print("Missing 'response' field in output")
51+
sys.exit(1)
52+
53+
response = output_content["response"]
54+
55+
# Check response structure
56+
if "blocks" not in response:
57+
print("Missing 'blocks' field in response")
58+
sys.exit(1)
59+
60+
blocks = response["blocks"]
61+
if not blocks or len(blocks) == 0:
62+
print("No response blocks found")
63+
sys.exit(1)
64+
65+
# Check for text content
66+
text_block = blocks[0]
67+
if "text" not in text_block:
68+
print("Missing 'text' field in response block")
69+
sys.exit(1)
70+
71+
response_text = text_block["text"]
72+
73+
# Check for tool calls (function execution)
74+
if "tool_calls" not in output_content:
75+
print("Missing 'tool_calls' field - function was not called")
76+
sys.exit(1)
77+
78+
tool_calls = output_content["tool_calls"]
79+
if not tool_calls or len(tool_calls) == 0:
80+
print("No tool calls found - function was not executed")
81+
sys.exit(1)
82+
83+
# Validate the specific function call
84+
first_call = tool_calls[0]
85+
expected_fields = ["tool_name", "tool_kwargs", "tool_id"]
86+
87+
for field in expected_fields:
88+
if field not in first_call:
89+
print(f"Missing '{field}' in tool call")
90+
sys.exit(1)
91+
92+
# Check if the correct function was called
93+
if first_call["tool_name"] != "may_research_company":
94+
print(f"Wrong function called: {first_call['tool_name']}, expected 'may_research_company'")
95+
sys.exit(1)
96+
97+
# Check function parameters
98+
tool_kwargs = first_call["tool_kwargs"]
99+
if "company_name" not in tool_kwargs:
100+
print("Missing 'company_name' parameter in function call")
101+
sys.exit(1)
102+
103+
company_name = tool_kwargs["company_name"]
104+
105+
# Check if response indicates successful approval
106+
if "research" not in response_text.lower() or "uipath" not in response_text.lower():
107+
print("Response doesn't indicate successful research approval")
108+
sys.exit(1)
109+
110+
print("All HITL-specific validations passed:")
111+
print(f"Agent response: {response_text}")
112+
print(f"Function called: {first_call['tool_name']}")
113+
print(f"Company researched: {company_name}")
114+
print(f"Tool call ID: {first_call['tool_id']}")
115+
116+
print("Simple HITL agent working correctly.")
117+
118+
except Exception as e:
119+
print(f"Error checking output: {e}")
120+
sys.exit(1)

0 commit comments

Comments
 (0)