Improve tool calling prompts, modify pre-req agent behavior

Signed-off-by: Kelly Abuelsaad <kaymar@gmail.com>

Author: kellyaa

a2a/git_issue_agent/a2a_agent.py

@@ -158,10 +158,14 @@ async def execute(self, context: RequestContext, event_queue: EventQueue):
         Returns:
             None
         """
-        if settings.JWKS_URI:
-            user_token = context.call_context.user._user.access_token
-        else:
-            user_token = settings.GITHUB_TOKEN
+        ###
+        # commenting this out for now since we have external github MCP.
+        # in the future we need to figure out the token exchange story for this scenario
+        #
+        #if settings.JWKS_URI:
+        #    user_token = context.call_context.user._user.access_token
+        #else:
+        user_token = settings.GITHUB_TOKEN
         user_input = [context.get_user_input()]
         task = context.current_task
         if not task:

a2a/git_issue_agent/git_issue_agent/agents.py

@@ -1,8 +1,8 @@
 from crewai import Agent, Crew, Process, Task
 from git_issue_agent.config import Settings
-from git_issue_agent.data_types import UserQueryJudgement
+from git_issue_agent.data_types import IssueSearchInfo
 from git_issue_agent.llm import CrewLLM
-from git_issue_agent.prompts import REPO_ID_BACKSTORY, TOOL_CALL_PROMPT
+from git_issue_agent.prompts import TOOL_CALL_PROMPT, INFO_PARSER_PROMPT
 class GitAgents():
 
     def __init__(self, config: Settings, issue_tools):
@@ -12,21 +12,24 @@ def __init__(self, config: Settings, issue_tools):
         # Pre-requisitie validator
         # ##################
         self.prereq_identifier = Agent(
-            role="Pre-requisite Judge",
-            goal="To determine whether a user has supplied enough information to find Github issues.",
-            backstory=REPO_ID_BACKSTORY,
+            role="Pre-requisite Extractor",
+            goal="To extract the information about github artifacts that a user is looking for",
+            backstory=INFO_PARSER_PROMPT,
             verbose=True,
             llm=self.llm.llm
         )
 
         self.prereq_identifier_task = Task(
             description=(
-                "User query: {request}"
+                "User query: {request}, \
+                Identified repository: {repo} \
+                Identified owner: {owner} \
+                Identified issue numbers: {issues}"
             ),
             agent=self.prereq_identifier,
-            output_pydantic=UserQueryJudgement,
+            output_pydantic=IssueSearchInfo,
             expected_output=(
-                "A judgement on whether a request from a user successfully identifies an organization or user that owns issues."
+                "A structured extraction of the relevant Github artifacts"
             ),
         )
 

a2a/git_issue_agent/git_issue_agent/data_types.py

@@ -5,7 +5,7 @@
 # Pydantic types for LLM response formats
 ############
 
-
-class UserQueryJudgement(BaseModel):
-    has_sufficient_information: bool = Field(False, description="Whether the user's query contains sufficient enough information to retrieve issues from github")
-    explanation: str = Field(None, description="A detailed explanation as to why you made the judgement")
+class IssueSearchInfo(BaseModel):
+    owner: str
+    repo: str
+    issue_numbers: list[int]

a2a/git_issue_agent/git_issue_agent/main.py

@@ -22,6 +22,7 @@ def __init__(self, config: Settings,
 
         self.agents = GitAgents(settings, mcp_toolkit)
         self.eventer = eventer
+        self.logger = logger or logging.getLogger(__name__)
 
     async def _send_event(self, message: str, final: bool = False):
         logger.info(message)
@@ -48,16 +49,19 @@ def extract_user_input(self, body):
     async def execute(self, user_input):
         query = self.extract_user_input(user_input)
         await self._send_event("🧐 Evaluating requirements...")
-        await self.agents.prereq_id_crew.kickoff_async(inputs={"request": query})
+        await self.agents.prereq_id_crew.kickoff_async(
+            inputs={"request": query, "repo": "", "owner": "", "issues": []}
+        )
         repo_id_task_output = self.agents.prereq_identifier_task.output.pydantic
 
-        if not repo_id_task_output.has_sufficient_information:
-            return repo_id_task_output.explanation
+        if repo_id_task_output.issue_numbers:
+            if not repo_id_task_output.owner or not repo_id_task_output.repo:
+                return "When supplying issue numbers, you must provide both a repository name and owner."
+        if repo_id_task_output.repo:
+            if not repo_id_task_output.owner:
+                return "When supplying a repository name, you must also provide an owner of the repo."
 
         await self._send_event("🔎 Searching for issues...")
-        await self.agents.crew.kickoff_async(inputs={"request": query})
+        await self.agents.crew.kickoff_async(inputs={"request": query, "owner": repo_id_task_output.owner, "repo": repo_id_task_output.repo, "issues": repo_id_task_output.issue_numbers})
         return self.agents.issue_query_task.output.raw
-
 
-        
-        

a2a/git_issue_agent/git_issue_agent/prompts.py

@@ -1,87 +1,101 @@
 TOOL_CALL_PROMPT = """
-You are a GitHub issue analyst with access to MCP tools that can search and list GitHub issues. Use them to gather facts before answering the user.
+You are a GitHub issue analyst with access to MCP tools that can search and list GitHub issues.
 
 YOUR JOB
-1) Decide whether the user’s request can be fulfilled with a tool from the catalog.
-2) When a tool is required, emit a tool call in the exact format below and nothing else.
-3) After tool results arrive, produce the final answer grounded strictly in those results.
-
-──────────────────────────────────────────────────────────────────────────────
-TOOL CALL FORMAT (emit only this block when calling a tool)
-
-<|tool_call|>
-[
+1. Decide whether the user’s request can be fulfilled with a tool from the catalog.
+2. When a tool is required, emit **only** a single tool call in the exact format below.
+3. After tool results arrive, produce the final answer grounded strictly in those results.
+
+────────────────────────────────────────────────────────
+⚙️ TOOL CALL PHASE
+You are a GitHub issue analyst with access to MCP tools that can search and list GitHub issues.
+
+MODES
+- If a tool is required, you MUST output EXACTLY the following four lines in this order:
+  1) Thought: <one short sentence>
+  2) Action: <one of [list_issue_types, list_issues, list_sub_issues, search_issues]>
+  3) Action Input: <a single-line JSON object with only the schema's keys/values>
+  4) Observation: <leave blank – this will be filled by the system>
+
+- After the Observation is provided by the system, output:
+  Thought: I now know the final answer
+  Final Answer: <answer grounded ONLY in the tool output>
+
+STRICT RULES FOR TOOL CALLS
+- Output NOTHING except those exact lines when calling a tool. No code fences, no XML, no extra braces.
+- The JSON after "Action Input:" MUST be valid and single-line. No trailing commas, no extra closing braces.
+- Use ONLY properties defined in the tool schema (required + optional). Exact key names and value types.
+- Use only one tool per call.
+
+CORRECT EXAMPLE
+Thought: The user provided owner and repo; list_issues fits.
+Action: list_issues
+Action Input: {"owner":"kagenti","repo":"kagenti"}
+Observation:
+
+INCORRECT EXAMPLES (do NOT do these)
+- <tool_call>{"name":"list_issues",...}</tool_call>
+- Action: {"name":"list_issues","arguments":{...}}
+- Action Input:
   {
-    "name": "<tool_name>",
-    "arguments": {
-      ... key: value pairs that match the tool's JSON Schema exactly ...
-    }
+    "owner":"kagenti",
+    "repo":"kagenti",
   }
-]
-
-Rules for arguments:
-- Use ONLY the properties defined in that tool’s schema (required + optional).
-- Use the exact key names and value types from the schema.
-- Do NOT wrap parameters in another "arguments" object.
-- Do NOT include extra keys or trailing commentary.
-- If a property has an enum, choose one value from the enum.
-- If required information is missing, do not call the tool—wait or respond `NO_TOOL`.
+- Action: list_issues
+  Action Input: {"owner":"kagenti","repo":"kagenti"}}
 
-──────────────────────────────────────────────────────────────────────────────
-FINAL ANSWER FORMAT (after you receive tool output)
+────────────────────────────────────────────────────────
+🧩 FINAL ANSWER PHASE
 
-- Base your answer ONLY on tool output or supplied context.
-- Clearly cite the relevant tool results when explaining the outcome.
-- If a tool failed or could not be called because inputs were missing, say so.
+After tool results arrive:
+- Produce a human-readable answer grounded only in the tool output.
+- Clearly cite or reference the tool results.
+- If a tool failed or inputs were missing, say so explicitly. Don't attempt to guess the answer.
 
-──────────────────────────────────────────────────────────────────────────────
+────────────────────────────────────────────────────────
 TOOL SELECTION GUIDELINES
 
 - Do **not** guess repository owners, names, or issue numbers.
 - Choose the most specific tool that aligns with the user’s intent.
 - Respect each tool’s required and optional parameters; format them exactly as expected.
 - Use only one tool call at a time.
 
+Here are some additional descriptions of the tools you will be provided, along with guidance on how to choose teh right one.
+
+**Search Issues:** Use this tool to find issues when you do not have a repository name. Optionally scope by owner or repo if provided.
 **List Issue Types:** Use only to enumerate available issue types for a specific organization.
-**List Issues:** Use only when both repository owner and exact repository name are provided. Optional filters: state, label, date, etc.
-**List Sub-Issues:** Use only when owner, repo, and issue number are given.
-**Search Issues:** Use for broader queries or when owner/repo are unknown. Optionally scope by owner or repo if provided.
+**List Issues:** Use only when both repository owner AND exact repository name are provided. Do not use unless you have both pieces of information. Optional filters: state, label, date, etc.
+**List Sub-Issues:** Use only when owner, repo, and issue number are all given.
+
 
 Decision rules:
-- Prefer list-style tools when the query targets a specific repo or issue.
-- Prefer search when the user’s scope is broad or unspecified.
+- Prefer search when the user’s scope is broad or unspecified, or you don't have a repository name.
+- Always use list-style tools when the query provides a repo name or issue number.
 - Never infer missing identifiers.
+- Never call a tool when you do not have all the required parameters.
 
 Examples:
 - “Open issues in `kagenti/agent-examples`” → list issues
 - “Find issues mentioning ‘timeout’ across all repos” → search issues
 - “Issue types for the `ibm` organization” → list issue types
 - “Sub-issues under #134 in `openai/triton`” → list sub-issues
+- "What issues are assigned to joe123?" → search issues
 
 TOOL USE RULES
 - Use only the tools provided here.
 - Cite tool outputs or provided context; do not add outside knowledge.
 """
 
+INFO_PARSER_PROMPT = """
+You are an analyst that will extract out information from a user's instruction/query to determine the following information, if it exists:
+- Github owner or organization
+- Github repository
+- Issue number(s)
 
-REPO_ID_BACKSTORY = """
-You are a judge determining whether a user’s request successfully identifies sufficient information to retrieve Github issues.
-
-For a request to count as a successful identification, it must:
-- ✅ Explicitly name the an owner, which can be:
-  - an organization name,
-  - a username
-
-✅ Positive Examples
-
-- "summarize open issues across the foo organization" → Valid because the foo organization is identified.
-- "kagenti/agent-examples" → Identifies the kagenti organization.
-- "foo in the bar organization" → Identifies bar organization.
-- "the kubernetes organization" → Identifies the kubernetes organization.
-
-❌ Negative Examples
-- "the kubernetes repo" → Invalid because it only specifies a repository name, not the owner or the organization.
-- "show me open issues" (with no owner or organization) → Invalid because no owner or organization is identified.
-
-Your task is to return whether the user’s request meets these criteria.
+Examples:
+- "summarize open issues across the foo organization" → Owner: foo, Repo: None, Issues: None
+- "kagenti/agent-examples" → Owner: kagenti, Repo: agent-examples, Issues: None
+- "foo in the bar organization" → Owner: bar, Repo: foo, Issues: None
+- "Search across all of github/github-mcp-server for open issues with bug" → Owner: github, Repo: github-mcp-server, Issues: None
+- "How long has issue 2 in modelcontextprotocol/servers been open?"  → Owner: modelcontextprotocol, Repo: servers, Issues: [2]
 """