Run a reproducer in a simple way

Author: majamassarini

Containerfile.mcp

@@ -1,5 +1,7 @@
 FROM fedora:42
 
+RUN dnf -y copr enable @testing-farm/stable
+
 # Install system dependencies
 RUN dnf -y install \
       python3 \
@@ -16,6 +18,7 @@ RUN dnf -y install \
       krb5-workstation \
       centpkg \
       git \
+      testing-farm \
     && dnf clean all
 
 # Install FastMCP

Makefile

@@ -69,6 +69,9 @@ run-backport-agent-c9s-standalone:
 		-e DRY_RUN=$(DRY_RUN) \
 		-e MOCK_JIRA=$(MOCK_JIRA) \
 		-e CVE_ID=$(CVE_ID) \
+		-e REPRODUCER_INFO_REPO_URL=$(REPRODUCER_INFO_REPO_URL) \
+		-e REPRODUCER_INFO_REPO_REF=$(REPRODUCER_INFO_REPO_REF) \
+		-e REPRODUCER_INFO_TEST=$(REPRODUCER_INFO_TEST) \
 		backport-agent-c9s
 
 .PHONY: run-backport-agent-c10s-standalone
@@ -81,6 +84,9 @@ run-backport-agent-c10s-standalone:
 		-e DRY_RUN=$(DRY_RUN) \
 		-e MOCK_JIRA=$(MOCK_JIRA) \
 		-e CVE_ID=$(CVE_ID) \
+		-e REPRODUCER_INFO_REPO_URL=$(REPRODUCER_INFO_REPO_URL) \
+		-e REPRODUCER_INFO_REPO_REF=$(REPRODUCER_INFO_REPO_REF) \
+		-e REPRODUCER_INFO_TEST=$(REPRODUCER_INFO_TEST) \
 		backport-agent-c10s
 
 .PHONY: run-backport-agent-standalone

agents/backport_agent.py

@@ -39,6 +39,7 @@
     Task,
 )
 from common.utils import redis_client, fix_await
+from common.models import ReproducerInfo
 from constants import I_AM_JOTNAR, CAREFULLY_REVIEW_CHANGES
 from observability import setup_observability
 from tools.commands import RunShellCommandTool
@@ -288,6 +289,16 @@ def get_prompt() -> str:
       Everything from the previous attempt has been reset. Start over, follow the instructions from the start
       and don't forget to fix the issue.
       {{/build_error}}
+      {{^reproducer_success}}
+      The reproducer test case failed with the following error:
+
+      {{reproducer_result}}
+
+      Keep in mind the error and try to fix the patches in a way that makes the reproducer test case pass.
+
+      You can inspect the reproducer test case at {{reproducer_info.git_url}}/blob/{{reproducer_info.git_ref}}/{{reproducer_info.test}}.
+      The test is located in the {{tmt_reproducer.git_ref}} branch of the {{tmt_reproducer.git_url}} repository.
+      {{/reproducer_success}}
     """
 
 
@@ -535,9 +546,13 @@ class State(PackageUpdateState):
         attempts_remaining: int = Field(default=max_build_attempts)
         used_cherry_pick_workflow: bool = Field(default=False)  # Track if cherry-pick was used
         incremental_fix_attempts: int = Field(default=0)  # Track how many times we tried incremental fix
+        build_url: str | None = Field(default=None)
+        reproducer_success: bool = Field(default=False)
+        reproducer_result: str | None = Field(default=None)
+        reproducer_info: ReproducerInfo | None = Field(default=None)
 
     async def run_workflow(
-        package, dist_git_branch, upstream_patches, jira_issue, cve_id, redis_conn=None
+        package, dist_git_branch, upstream_patches, jira_issue, cve_id, redis_conn=None, reproducer_info=None
     ):
         local_tool_options["working_directory"] = None
 
@@ -603,6 +618,9 @@ async def run_backport_agent(state):
                             cve_id=state.cve_id,
                             upstream_patches=state.upstream_patches,
                             build_error=state.build_error,
+                            reproducer_success=state.reproducer_success,
+                            reproducer_result=state.reproducer_result,
+                            reproducer_info=state.reproducer_info,
                         ),
                     ),
                     expected_output=BackportOutputSchema,
@@ -663,6 +681,9 @@ async def fix_build_error(state):
                                 cve_id=state.cve_id,
                                 upstream_patches=state.upstream_patches,
                                 build_error=state.build_error,
+                                reproducer_success=state.reproducer_success,
+                                reproducer_result=state.reproducer_result,
+                                reproducer_info=state.reproducer_info,
                             ),
                         ),
                         expected_output=BackportOutputSchema,
@@ -736,10 +757,14 @@ async def run_build_agent(state):
                 if build_result.success:
                     # Build succeeded - reset incremental fix counter for potential future failures
                     state.incremental_fix_attempts = 0
-                    return "update_release"
+                    state.build_url = build_result.url
+                    if state.reproducer_info:
+                        return "run_reproducer_test_case"
+                    else:
+                        return "update_release"
                 if build_result.is_timeout:
                     logger.info(f"Build timed out for {state.jira_issue}, proceeding")
-                    return "update_release"
+                    return "run_testing_farm_test"
                 state.attempts_remaining -= 1
                 if state.attempts_remaining <= 0:
                     state.backport_result.success = False
@@ -878,6 +903,23 @@ async def create_merge_request_checklist(state):
             async def add_fusa_label(state):
                 return await PackageUpdateStep.add_fusa_label(state, "comment_in_jira", dry_run=dry_run, gateway_tools=gateway_tools)
 
+            async def run_reproducer_test_case(state):
+                state.reproducer_success, state.reproducer_result = await tasks.run_tool(
+                    "run_testing_farm_test",
+                    git_url=state.reproducer_info.git_url,
+                    git_ref=state.reproducer_info.git_ref,
+                    path_to_test=state.reproducer_info.test,
+                    package=state.build_url,
+                    compose=state.dist_git_branch.upper() + "-Nightly",
+                    available_tools=gateway_tools,
+                )
+                if state.reproducer_success:
+                    logger.info(f"Reproducer result: {state.reproducer_result}")
+                    return "update_release"
+                else:
+                    logger.error(f"Reproducer failed: {state.reproducer_result}")
+                    return "run_backport_agent"
+
             async def comment_in_jira(state):
                 if dry_run:
                     return Workflow.END
@@ -903,6 +945,7 @@ async def comment_in_jira(state):
             workflow.add_step("run_backport_agent", run_backport_agent)
             workflow.add_step("fix_build_error", fix_build_error)
             workflow.add_step("run_build_agent", run_build_agent)
+            workflow.add_step("run_reproducer_test_case", run_reproducer_test_case)
             workflow.add_step("update_release", update_release)
             workflow.add_step("stage_changes", stage_changes)
             workflow.add_step("run_log_agent", run_log_agent)
@@ -919,6 +962,9 @@ async def comment_in_jira(state):
                     upstream_patches=upstream_patches,
                     jira_issue=jira_issue,
                     cve_id=cve_id,
+                    reproducer_info=reproducer_info,
+                    reproducer_success=False,
+                    reproducer_result=None,
                 ),
             )
             return response.state
@@ -931,13 +977,24 @@ async def comment_in_jira(state):
     ):
         upstream_patches = upstream_patches_raw.split(",")
         logger.info("Running in direct mode with environment variables")
+        reproducer_info_repo_url = os.getenv("REPRODUCER_INFO_REPO_URL", None)
+        reproducer_info_repo_ref = os.getenv("REPRODUCER_INFO_REPO_REF", None)
+        reproducer_info_test = os.getenv("REPRODUCER_INFO_TEST", None)
+        reproducer_info = None
+        if reproducer_info_repo_url and reproducer_info_repo_ref and reproducer_info_test:
+            reproducer_info = ReproducerInfo(
+                git_url=reproducer_info_repo_url,
+                git_ref=reproducer_info_repo_ref,
+                test=reproducer_info_test,
+            )
         state = await run_workflow(
             package=package,
             dist_git_branch=branch,
             upstream_patches=upstream_patches,
             jira_issue=jira_issue,
             cve_id=os.getenv("CVE_ID", None),
             redis_conn=None,
+            reproducer_info=reproducer_info,
         )
         logger.info(f"Direct run completed: {state.backport_result.model_dump_json(indent=4)}")
         return
@@ -967,7 +1024,7 @@ async def comment_in_jira(state):
             logger.info(
                 f"Processing backport for package: {backport_data.package}, "
                 f"JIRA: {backport_data.jira_issue}, branch: {dist_git_branch}, "
-                f"attempt: {task.attempts + 1}"
+                f"attempt: {task.attempts + 1}, reproducer info: {backport_data.reproducer_info}"
             )
 
             async def retry(task, error):
@@ -1000,6 +1057,7 @@ async def retry(task, error):
                     jira_issue=backport_data.jira_issue,
                     cve_id=backport_data.cve_id,
                     redis_conn=redis,
+                    reproducer_info=backport_data.reproducer_info,
                 )
                 logger.info(
                     f"Backport processing completed for {backport_data.jira_issue}, " f"success: {state.backport_result.success}"

agents/build_agent.py

@@ -35,6 +35,7 @@ def get_instructions() -> str:
       just return the error message. Otherwise, start with `builder-live.log` and try to identify
       the build failure. If not found, try the same with `root.log`. Summarize the findings
       and return them as `error`. If the build failed due to a build timeout, set `is_timeout` to `true` in your output.
+      If the build succeeded, return the URL to the built package in `url`.
 
       General instructions:
 

agents/triage_agent.py

@@ -253,7 +253,22 @@ def render_prompt(input: InputSchema) -> str:
            is clarification-needed
          * This is the correct choice when you are sure a problem exists but cannot find the solution yourself
 
-         2.5 Set the Jira fields as per the instructions below.
+         2.5. If the chosen action is backport then search for the issue reproducer in the issue comments.
+         You need to extract the following information to say you have found the issue reproducer:
+            - Git URL to the issue reproducer; a git repo stored in https://gitlab.com/redhat/rhel/tests/
+            - Git reference, it can be omitted and in this case you will use "main" as the reference.
+            - Test to run, usually a path like folder/test_name
+            - You need to say you have found the issue reproducer and provide the information in the JSON format.
+              The JSON format should be like this:
+              ```json
+              {
+                "git_url": "https://gitlab.com/redhat/rhel/tests/test_repo.git",
+                "git_ref": "main",
+                "test": "folder/test_name"
+              }
+              ```
+
+         2.6. Set the Jira fields as per the instructions below.
 
       3. **No Action**
          A No Action decision is appropriate for issues that are NOT bugs or CVEs requiring code fixes:
@@ -410,7 +425,12 @@ async def run_triage_analysis(state):
                             "justification": "This patch fixes the bug by doing X, Y, and Z.",
                             "jira_issue": "RHEL-12345",
                             "cve_id": "CVE-1234-98765",
-                            "fix_version": "rhel-X.Y.Z"
+                            "fix_version": "rhel-X.Y.Z",
+                            "reproducer_info": {{
+                              "git_url": "https://gitlab.com/redhat/rhel/tests/test_repo.git",
+                              "git_ref": "main",
+                              "test": "folder/test_name"
+                            }}
                           }}
                         }}
                         ```

common/models.py

@@ -88,6 +88,12 @@ class RebaseOutputSchema(BaseModel):
 # Backport Agent Schemas
 # ============================================================================
 
+class ReproducerInfo(BaseModel):
+    """Data for TMT resolution."""
+    git_url: str = Field(description="Git URL to the TMT reproducer")
+    git_ref: str = Field(description="Git reference to the TMT reproducer")
+    test: str = Field(description="Test to run the TMT reproducer")
+
 class BackportInputSchema(BaseModel):
     """Input schema for the backport agent."""
     local_clone: Path = Field(description="Path to the local clone of forked dist-git repository")
@@ -99,6 +105,9 @@ class BackportInputSchema(BaseModel):
     upstream_patches: list[str] = Field(
         description="List of URLs to upstream patches that were validated using the PatchValidator tool")
     build_error: str | None = Field(description="Error encountered during package build")
+    reproducer_info: ReproducerInfo | None = Field(description="Information about the TMT reproducer", default=None)
+    reproducer_success: bool = Field(default=False, description="Whether the reproducer test case passed")
+    reproducer_result: str | None = Field(description="Result of the reproducer test case", default=None)
 
 
 class BackportOutputSchema(BaseModel):
@@ -129,7 +138,6 @@ class RebaseData(BaseModel):
     jira_issue: str = Field(description="Jira issue identifier")
     fix_version: str | None = Field(description="Fix version in Jira (e.g., 'rhel-9.8')", default=None)
 
-
 class BackportData(BaseModel):
     """Data for backport resolution."""
     package: str = Field(description="Package name")
@@ -139,6 +147,7 @@ class BackportData(BaseModel):
     jira_issue: str = Field(description="Jira issue identifier")
     cve_id: str | None = Field(description="CVE identifier", default=None)
     fix_version: str | None = Field(description="Fix version in Jira (e.g., 'rhel-9.8')", default=None)
+    reproducer_info: ReproducerInfo | None = Field(description="Reproducer information", default=None)
 
 
 class ClarificationNeededData(BaseModel):
@@ -246,6 +255,7 @@ class BuildInputSchema(BaseModel):
 class BuildOutputSchema(BaseModel):
     """Output schema for the build agent."""
     success: bool = Field(description="Whether the build was successfully completed")
+    url: str | None = Field(description="URL to the built package", default=None)
     error: str | None = Field(description="Specific details about an error")
     is_timeout: bool = Field(default=False, description="Whether the build failed due to a timeout")
 

mcp_server/gateway.py

@@ -9,13 +9,14 @@
 import gitlab_tools
 import jira_tools
 import lookaside_tools
+import testing_farm_tools
 
 
 mcp = FastMCP(
     name="MCP Gateway",
     tools=[
         coroutine
-        for module in [copr_tools, distgit_tools, gitlab_tools, jira_tools, lookaside_tools]
+        for module in [copr_tools, distgit_tools, gitlab_tools, jira_tools, lookaside_tools, testing_farm_tools]
         for name, coroutine in inspect.getmembers(module, inspect.iscoroutinefunction)
         if coroutine.__module__ == module.__name__
         and not name.startswith("_")

mcp_server/testing_farm_tools.py

@@ -0,0 +1,56 @@
+import asyncio
+import logging
+import shlex
+from typing import Annotated, Tuple
+
+from fastmcp.exceptions import ToolError
+from pydantic import Field
+
+logger = logging.getLogger(__name__)
+
+
+async def run_testing_farm_test(
+    git_url: Annotated[str, Field(description="Git URL to the test repository")],
+    git_ref: Annotated[str, Field(description="Git reference to the test repository")],
+    path_to_test: Annotated[str, Field(description="Path to the test to run")],
+    package: Annotated[str, Field(description="Package URL to be installed in the test environment")],
+    compose: Annotated[str, Field(description="Testing Farm compose to use")],
+) -> Tuple[bool, str]:
+    """Runs the specified testing-farm test and returns True if the test passed, False otherwise."""
+
+    tmt_prepare = f'--insert --how install --package {shlex.quote(package)}'
+
+    # Build the command arguments
+    cmd = [
+        "testing-farm",
+        "request",
+        "--tmt-prepare", tmt_prepare,
+        "--compose", compose,
+        "--git-ref", git_ref,
+        "--git-url", git_url,
+        "--test", path_to_test,
+    ]
+
+    logger.info(f"Running testing-farm command: {' '.join(shlex.quote(arg) for arg in cmd)}")
+
+    try:
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
+        stdout, stderr = await process.communicate()
+
+        if process.returncode == 0:
+            msg = f"Testing Farm test passed: \n stdout {'='*60}\n {stdout.decode()}\n {'='*60}\n stderr {'='*60}\n {stderr.decode()}\n {'='*60}"
+            logger.info(msg)
+            return True, msg
+        else:
+            msg = f"Testing Farm test failed (exit code {process.returncode}): \n stdout {'='*60}\n {stdout.decode()}\n {'='*60}\n stderr {'='*60}\n {stderr.decode()}\n {'='*60}"
+            logger.error(msg)
+            return False, msg
+
+    except Exception as e:
+        logger.error(f"Failed to run testing-farm command: {e}")
+        raise ToolError(f"Failed to run testing-farm test: {e}") from e

templates/mcp-gateway.env

@@ -6,3 +6,5 @@ GITLAB_TOKEN=
 JIRA_URL=https://issues.redhat.com
 # JIRA profile -> Create token
 JIRA_TOKEN=
+
+TESTING_FARM_API_TOKEN=