diff --git a/Makefile b/Makefile index eccad8e1..a66affc6 100644 --- a/Makefile +++ b/Makefile @@ -283,6 +283,14 @@ process-erratum: $(COMPOSE_SUPERVISOR) run --rm \ supervisor python -m ymir.supervisor.main $(DEBUG_FLAG) $(IGNORE_NEEDS_ATTENTION_FLAG) $(DRY_RUN_FLAG) process-erratum $(ERRATA_ID) +.PHONY: run-issue-verification-agent-standalone +run-issue-verification-agent-standalone: + $(COMPOSE_AGENTS) run --rm \ + -e JIRA_ISSUE=$(JIRA_ISSUE) \ + -e DRY_RUN=$(DRY_RUN) \ + -e IGNORE_NEEDS_ATTENTION=$(IGNORE_NEEDS_ATTENTION) \ + issue-verification-agent + .PHONY: run-preliminary-testing-agent-standalone run-preliminary-testing-agent-standalone: $(COMPOSE_AGENTS) run --rm \ diff --git a/README-agents.md b/README-agents.md index 43ff87eb..6b5c9cfa 100644 --- a/README-agents.md +++ b/README-agents.md @@ -11,6 +11,7 @@ Three agents process tasks through Redis queues: - **Triage Agent**: Analyzes JIRA issues and determines resolution path. It uses title, description, fields, and comments to find out root cause of the issue. It can ask for clarification, create tasks for other agents or may take no action if not needed. - **Rebase Agent**: Updates packages to newer upstream versions. A Rebase is only to be chosen when the issue explicitly instructs you to "rebase" or "update". It looks for upstream references that are linked, attached and present in the description or comments in the issue. - **Backport Agent**: Applies specific fixes/patches to packages. It looks for patches that are linked, attached and present in the description or comments in the issue. It tries to apply the patch and resolve any conflicts that may arise during the backport process. +- **Issue Verification Agent**: Manages the post-fix lifecycle of a JIRA issue — from merged MR through errata creation, testing analysis, and status transitions to RELEASE_PENDING. Migrated from the supervisor's `IssueHandler`. ## Dry run mode diff --git a/README-supervisor.md b/README-supervisor.md index 9781234d..818621a9 100644 --- a/README-supervisor.md +++ b/README-supervisor.md @@ -38,6 +38,10 @@ This will have to be repeated when the ticket expires. (Exporting the kcm-cache socket to the container seems better, but I wasn't able to get it to work due to various difficult to work around permission issues.) +## Note on IssueHandler migration + +The `IssueHandler` has been migrated to `ymir/agents/issue_verification_agent.py` as part of the BeeAI Framework refactoring. The supervisor still handles errata processing via `ErratumHandler`. For processing individual issues, prefer using the new agent directly (see README-agents.md). + ## Processing a single issue or erratum To process a single issue or erratum, you can run: diff --git a/agents_as_skills/README.md b/agents_as_skills/README.md index 364d1092..62a928af 100644 --- a/agents_as_skills/README.md +++ b/agents_as_skills/README.md @@ -16,6 +16,24 @@ This directory contains Ymir workflows packaged as **AI coding assistant skills* For installation instructions (skill setup and MCP tool configuration), see the [Skills Installation Guide](https://github.com/packit/ai-workflows/blob/main/skills_installation.md). +## Available skills + +- `backport/` — Backport fix agent +- `preliminary_testing/` — Preliminary testing agent +- `rebase/` — Rebase fix agent +- `rebuild/` — Rebuild agent +- `triage/` — Triage agent +- `issue_verification/` — Issue verification agent (post-fix lifecycle management) + +## Available skills + +- `backport/` — Backport fix agent +- `preliminary_testing/` — Preliminary testing agent +- `rebase/` — Rebase fix agent +- `rebuild/` — Rebuild agent +- `triage/` — Triage agent +- `issue_verification/` — Issue verification agent (post-fix lifecycle management) + ## How to build ```bash diff --git a/agents_as_skills/issue_verification/SKILL.md b/agents_as_skills/issue_verification/SKILL.md new file mode 100644 index 00000000..8bd7493e --- /dev/null +++ b/agents_as_skills/issue_verification/SKILL.md @@ -0,0 +1,358 @@ +--- +description: > + Runs the Issue Verification workflow for a JIRA issue, managing the lifecycle + from merged MR through errata creation, testing analysis, and status transitions. +arguments: + - name: jira_issue + description: "JIRA issue key (e.g. RHEL-12345)" + required: true + - name: dry_run + description: "If true, skip all JIRA modifications (label changes, comments, status transitions). Default: false" + required: false + - name: ignore_needs_attention + description: "If true, process the issue even if it has the ymir_needs_attention label. Default: false" + required: false +--- + +# Issue Verification Skill + +You are the issue verification agent for Project Ymir. Your task is to manage the lifecycle of a RHEL JIRA issue after a fix has been backported or rebased — from the merge of a fix MR through errata creation, final testing analysis, and status transitions toward release. + +## Input Arguments + +- `jira_issue`: {{jira_issue}} — The JIRA issue key to process +- `dry_run`: {{dry_run}} — When true, skip all JIRA modifications +- `ignore_needs_attention`: {{ignore_needs_attention}} — When true, process even if the issue has the `ymir_needs_attention` label + +## Tools + +This skill uses the following tools. Do not restrict tool usage — use any tool available as needed. + +**MCP Tools (called via MCP gateway):** +- `get_jira_details` — Fetch full details of a JIRA issue +- `edit_jira_labels` — Add or remove labels on a JIRA issue +- `add_jira_comment` — Post a comment to a JIRA issue +- `change_jira_status` — Transition a JIRA issue to a new status +- `update_jira_comment` — Update an existing comment on a JIRA issue +- `add_jira_attachments` — Add file attachments to a JIRA issue +- `search_gitlab_project_mrs` — Search for merge requests in a GitLab project +- `get_erratum` — Get erratum details including comments and status +- `get_erratum_build_nvr` — Get the previous build NVR for a component from an erratum +- `get_testing_farm_request` — Get Testing Farm request status and results +- `reproduce_testing_farm_request` — Reproduce a Testing Farm test run with a different build NVR + +**Other:** +- `analyze_ewa_testrun` — Analyze an EWA (Errata Workflow Automation) TCMS test run +- `get_jira_attachment` — Download a JIRA issue attachment by filename +- `read_logfile` — Read a Testing Farm log file +- `search_resultsdb` — Search ResultsDB for test results +- WebFetch for fetching web content (e.g., Testing Farm artifacts) + +## Constants + +**JIRA Custom Field IDs:** +- **Errata Link**: `customfield_10418` (or `customfield_10626` as fallback) +- **Fixed in Build**: `customfield_10578` — contains the build NVR +- **Test Coverage**: `customfield_10638` — multi-value field; valid values: `Manual`, `Automated`, `RegressionOnly`, `New Test Coverage` +- **Preliminary Testing**: `customfield_10879` — single-value field with a `value` key (e.g., `"Pass"`) +- **AssignedTeam**: `customfield_10371` + +**JIRA Labels:** +- `ymir_needs_attention` — Issue needs human attention +- `ymir_backported` — Fix was backported +- `ymir_rebased` — Package was rebased +- `ymir_merged` — MR was merged +- `ymir_reproducing_tests` — Baseline test reproduction is in progress + +**GitLab Groups to search for MRs:** +- `redhat/rhel/rpms` +- `redhat/centos-stream/rpms` + +**Issue Statuses:** +- `New`, `Planning`, `In Progress`, `Integration`, `Release Pending`, `Closed` + +## Attention Template + +When flagging an issue for attention, add the `ymir_needs_attention` label and post a private comment using this format: + +``` +{panel:title=Project Ymir: ATTENTION NEEDED|borderStyle=solid|borderColor=#CC0000|titleBGColor=#FFF5F5|bgColor=#FFFEF0} + + +Please resolve this and remove the {ymir_needs_attention} flag. +{panel} + + +``` + +Where `` is the specific reason attention is needed, and `` is an optional detailed analysis comment appended after the panel. + +To flag attention: +1. Call `edit_jira_labels` with `issue_key` = `{{jira_issue}}` and `labels_to_add` = `["ymir_needs_attention"]`. +2. Call `add_jira_comment` with `issue_key` = `{{jira_issue}}`, `comment` = the formatted attention comment, and `private` = true. + +## Workflow + +Execute the following steps in order. Track state across steps. + +### Step 1: Fetch and Validate Issue + +1. Call `get_jira_details` with `issue_key` = `{{jira_issue}}`. +2. Save the full issue data for later steps. +3. Extract and check the following from the issue `fields`: + +**Check `ymir_needs_attention` label:** +- Extract `labels` from the issue fields. +- If the label `ymir_needs_attention` is present AND `ignore_needs_attention` is false: + - End the workflow with status `"Issue has the ymir_needs_attention label"` and reschedule_in = -1. + +**Check component count:** +- Extract `components` from the issue fields. +- If the number of components is not exactly 1: + - If `dry_run` is false, flag attention with why = `"This issue has multiple components. Ymir only handles issues with single component currently."` (no details comment). + - End the workflow. + +If all checks pass, proceed to Step 2. + +### Step 2: Check Errata Status + +Extract the errata link from `customfield_10418` (or fallback `customfield_10626`). + +- If the errata link is **null/absent** → proceed to Step 3 (Before Errata). +- If the errata link **exists** → proceed to Step 4 (After Errata). + +### Step 3: Before Errata (no errata link) + +This step handles issues where a fix MR has been merged but no erratum has been created yet. + +**3.1. Check for target labels:** +- If the issue does NOT have any of the labels `ymir_backported`, `ymir_rebased`, or `ymir_merged`: + - End the workflow with status `"Issue without target labels: "` and reschedule_in = -1. + +**3.2. Check and add merged label:** +- If the issue does NOT have the `ymir_merged` label: + - Search for merged MRs in both GitLab groups (`redhat/rhel/rpms/` and `redhat/centos-stream/rpms/`) using `search_gitlab_project_mrs` with `search` = `{{jira_issue}}` and `state` = `"merged"`. + - If a merged MR is found: + - If `dry_run` is false, add the `ymir_merged` label with a comment: `"A [merge request|] resolving this issue has been merged; waiting for errata creation and final testing."` + +**3.3. Check merged status after labeling attempt:** +- If the issue STILL does not have the `ymir_merged` label: + - End the workflow with status `"No merged MR found, reschedule it for 3 hours"` and reschedule_in = 10800 (3 hours). + +**3.4. Check time since merge:** +- Get the latest merged timestamp from all merged MRs (search both GitLab groups again). +- If less than 24 hours have passed since the latest merge: + - End the workflow with status `"Wait for the associated erratum to be created"` and reschedule_in = 3600 (1 hour). +- If more than 24 hours have passed: + - Flag attention with why = `"A merge request was merged for this issue more than 24 hours ago but no errata was created. Please investigate and look for gating failures or other reasons that might have blocked errata creation."` + - End the workflow. + +### Step 4: After Errata (errata link exists) + +This step handles issues where an erratum has been created. + +**4.1. Check Fixed in Build:** +- Extract `customfield_10578` (Fixed in Build) from the issue fields. +- If null: + - Flag attention with why = `"Issue has errata_link but no fixed_in_build"`. + - End the workflow. + +**4.2. Check Preliminary Testing:** +- Extract the value from `customfield_10879` (Preliminary Testing). +- If the field is a dict, read its `value` key. +- If the value is NOT `"Pass"`: + - Flag attention with why = `"Issue does not have Preliminary Testing set to Pass - this should have happened before the gitlab pull request was merged"`. + - End the workflow. + +**4.3. Check Test Coverage:** +- Extract `customfield_10638` (Test Coverage) from the issue fields. +- If the field is null or an empty list: + - Flag attention with why = `"Issue does not have Test Coverage set - this should have happened before the gitlab pull request was merged"`. + - End the workflow. + +**4.4. Add merged label:** +- Even in post-errata state, attempt to add the `ymir_merged` label using the same logic as Step 3.2 (for JIRA dashboard purposes). + +**4.5. Branch on issue status:** + +- **New, Planning, or In Progress:** + - If `dry_run` is false: + - Call `change_jira_status` with `issue_key` = `{{jira_issue}}` and `status` = `"Integration"`. + - Call `add_jira_comment` with a private comment: `"*Changing status from => Integration*\n\nPreliminary testing has passed, moving to Integration"`. + - End the workflow with reschedule_in = 0. + +- **Integration:** + - If the issue has the `ymir_reproducing_tests` label → proceed to Step 6 (Check Reproduction). + - Otherwise → proceed to Step 5 (Analyze Testing). + +- **Release Pending or Closed:** + - End the workflow with status `"Issue status is "` and reschedule_in = -1. + +- **Any other status:** + - Report an error: `"Unknown issue status: "`. + +### Step 5: Analyze Testing + +This step performs a thorough analysis of test results for the issue. You act as a testing analyst. + +**5.1. Fetch erratum data:** +- Call `get_erratum` with `erratum_id` = the errata link from the issue, and `full` = true. +- Save the full erratum data including comments. + +**5.2. Check for previous baseline test analysis:** +- Search through the issue comments (in reverse order) for a comment matching the pattern `".*failed tests with previous build (.*):"`. +- If found, this means baseline test reproduction was previously completed. Set `after_baseline` = true. +- If not found, set `after_baseline` = false. + +**5.3. Determine test location info:** +- The component's tests may be triggered by NEWA (New Errata Workflow Automation) or EWA (Errata Workflow Automation). +- NEWA posts comments to the erratum with links to JIRA issues containing test results. +- EWA posts comments to the erratum with links to TCMS Test Runs. +- If tests are supposed to be started by NEWA but no NEWA comments exist, the component may only use NEWA for RHEL10 — in that case, check TCMS test runs from EWA. + +**5.4. Analyze test results:** + +Use available tools (`get_jira_attachment`, `read_logfile`, `search_resultsdb`, `analyze_ewa_testrun`) to find and analyze test results. + +**IMPORTANT:** OSCI gating tests run as part of the GitLab MR pipeline and do NOT constitute final testing. You must find evidence of full integration and regression testing triggered by NEWA or EWA (posted as comments on the erratum) before concluding tests have passed. If only OSCI gating results are available, the state is `tests-pending`. + +You cannot assume that tests have passed just because a comment says they have finished — you must check the actual test results in the JIRA issue or TCMS. Verify that the JIRA issue or TCMS Test Run is the correct one for the latest build in the erratum. + +If the erratum is in QE status, its last status transition was more than 6 hours ago, and there's no evidence of tests running or completed, assume tests will not run automatically → state is `tests-not-running`. + +**If `after_baseline` is true:** +- Previous analysis identified failing test runs that have been reproduced with a baseline build. +- Read the issue comments and attachments to find the baseline test comparison results. +- Check log files (2-3 per architecture) to verify failures are consistent between runs. +- If all failures in the new build also occurred with the baseline build and are consistent, classify as `tests-waived`. +- If failures appear to be regressions, classify as `tests-failed`. +- Do not use `tests-waived` if tests could not be run on the new build. + +**5.5. Act on the testing state:** + +Based on your analysis, determine the testing state and take action: + +- **tests-passed:** + - If `dry_run` is false: + - Call `change_jira_status` with `status` = `"Release Pending"`. + - Call `add_jira_comment` with a private comment describing what was tested, with links to results. + - End the workflow with reschedule_in = -1. + +- **tests-waived:** + - Same as tests-passed — transition to `"Release Pending"` with a comment explaining why failures are not considered regressions. + - End the workflow with reschedule_in = -1. + +- **tests-failed:** + - If the analysis identified specific failed Testing Farm request IDs AND this is NOT a repeat of already-known failures: + - Attempt to start baseline test reproduction: + 1. Call `get_erratum_build_nvr` with `erratum_id` and `component` to get the previous build NVR. + 2. If no previous build NVR is available: + - Flag attention with why = `"Tests failed - see details below. Cannot start reproduction with previous build - error finding previous build NVR."` and include the analysis comment. + 3. For each failed Testing Farm request ID: + - Call `get_testing_farm_request` to get the full request details. + - Call `reproduce_testing_farm_request` with `request_id` and `build_nvr` = the previous build NVR. + 4. Add the `ymir_reproducing_tests` label with a comment showing a table of original vs. baseline requests. + 5. End the workflow with reschedule_in = 1200 (20 minutes). + - If reproduction cannot be started or this is a repeat failure: + - Flag attention with why = `"Tests failed - see details below"` and include the analysis comment. + - End the workflow. + +- **tests-error:** + - Flag attention with why = `"An error occurred during testing - see details below"` and include the analysis comment. + - End the workflow. + +- **tests-pending:** + - End the workflow with status `"Tests are pending"` and reschedule_in = 1200 (20 minutes). + +- **tests-running:** + - End the workflow with status `"Tests are running"` and reschedule_in = 1200 (20 minutes). + +- **tests-not-running:** + - Flag attention with why = `"Tests aren't running - see details below"` and include the analysis comment. + - End the workflow. + +### Step 6: Check Reproduction + +This step checks whether baseline test reproduction has completed. + +**6.1. Parse baseline test data from comments:** +- Search through the issue comments (in reverse order) for a comment containing the baseline test reproduction table. +- The table has the pattern: `".*failed tests with previous build :"` followed by a table with columns: Architecture, Original Request, Request With Old Build, State/Result. +- Extract the failed request ID and baseline request ID from each row. +- If no baseline test data is found in comments: + - Flag attention with why = `"Issue has ymir_reproducing_tests label but cannot parse baseline tests from comments"`. + - End the workflow. + +**6.2. Check if all baseline tests have settled:** +- For each baseline request ID, call `get_testing_farm_request` to check its state. +- A request has "settled" if its state is `complete`, `error`, or `canceled`. +- If any baseline request has NOT settled: + - End the workflow with status `"Waiting for baseline tests to complete"` and reschedule_in = 1200 (20 minutes). + +**6.3. Generate comparison attachments:** +- For each pair (failed request, baseline request): + - If both have xunit result URLs, fetch and compare the xunit results. + - Create a comparison attachment named `comparison---.toml`. + - Upload all comparison attachments using `add_jira_attachments`. + +**6.4. Update the comment and remove label:** +- Remove the `ymir_reproducing_tests` label. +- Update the existing baseline tests comment (using `update_jira_comment` with the comment_id) to include: + - The original failure comment. + - An updated table with a Result column and Comparison column linking to the attachments. +- End the workflow with status `"Baseline tests are complete, will analyze results"` and reschedule_in = 0 (immediate re-run, which will go back through Step 5 with `after_baseline` = true). + +--- + +## Output Schema + +The final output must be a JSON object: + +```json +{ + "status": "Description of what happened during the workflow run", + "reschedule_in": -1 +} +``` + +### reschedule_in Values + +- **-1**: Do not reschedule — terminal state (needs_attention flagged, Release Pending, Closed, or no target labels) +- **0**: Reschedule immediately — workflow should be re-run (e.g., after baseline tests complete) +- **1200**: Reschedule in 20 minutes — waiting for tests to run or baseline reproduction to complete +- **3600**: Reschedule in 1 hour — waiting for erratum creation +- **10800**: Reschedule in 3 hours — waiting for merged MR to appear + +### Examples + +**Issue moved to Integration:** +```json +{ + "status": "Preliminary testing has passed, moving to Integration", + "reschedule_in": 0 +} +``` + +**Tests passed, moved to Release Pending:** +```json +{ + "status": "Final testing has passed.", + "reschedule_in": -1 +} +``` + +**Waiting for erratum creation:** +```json +{ + "status": "Wait for the associated erratum to be created", + "reschedule_in": 3600 +} +``` + +**Attention flagged:** +```json +{ + "status": "Tests failed - see details below", + "reschedule_in": -1 +} +``` diff --git a/compose.yaml b/compose.yaml index 7368edc3..48b6bbea 100644 --- a/compose.yaml +++ b/compose.yaml @@ -164,6 +164,13 @@ services: command: ["python", "-m", "ymir.agents.preliminary_testing_agent"] profiles: ["agents"] + issue-verification-agent: + <<: *beeai-agent-c10s + environment: + <<: *beeai-env + command: ["python", "-m", "ymir.agents.issue_verification_agent"] + profiles: ["agents"] + triage-agent-e2e-tests: <<: *beeai-agent-c10s environment: diff --git a/ymir/agents/issue_verification_agent.py b/ymir/agents/issue_verification_agent.py new file mode 100644 index 00000000..36eec534 --- /dev/null +++ b/ymir/agents/issue_verification_agent.py @@ -0,0 +1,914 @@ +import asyncio +import logging +import os +import sys +import traceback +from datetime import UTC, datetime, timedelta +from typing import Any + +from beeai_framework.errors import FrameworkError +from beeai_framework.memory import UnconstrainedMemory +from beeai_framework.template import PromptTemplate, PromptTemplateInput +from beeai_framework.tools.think import ThinkTool +from beeai_framework.workflows import Workflow +from pydantic import BaseModel, Field + +from ymir.agents.observability import setup_observability +from ymir.agents.reasoning_agent import ReasoningAgent +from ymir.agents.utilities.baseline_tests import BaselineTests +from ymir.agents.utils import ( + get_agent_execution_config, + get_chat_model, + get_tool_call_checker_config, + is_reasoning_enabled, + mcp_tools, + run_tool, +) +from ymir.common.constants import DATETIME_MIN_UTC, GITLAB_GROUPS, JiraLabels +from ymir.common.models import ( + FullErratum, + FullIssue, + GitlabMergeRequest, + GitlabMergeRequestState, + IssueStatus, + JiraComment, + PreliminaryTesting, + TestCoverage, + TestingState, + WorkflowResult, +) +from ymir.tools.privileged.jira import GetJiraAttachmentTool +from ymir.tools.unprivileged.analyze_ewa_testrun import AnalyzeEwaTestRunTool +from ymir.tools.unprivileged.read_logfile import ReadLogfileTool +from ymir.tools.unprivileged.read_readme import ReadReadmeTool +from ymir.tools.unprivileged.search_resultsdb import SearchResultsdbTool + +logger = logging.getLogger(__name__) + +WAIT_DELAY = 20 * 60 # 20 minutes +MERGE_CHECK_DELAY = 3 * 60 * 60 # 3 hours +ERRATA_WAIT_DELAY = 60 * 60 # 1 hour + +ATTENTION_TEMPLATE = ( + "{{panel:title=Project Ymir: ATTENTION NEEDED|" + "borderStyle=solid|borderColor=#CC0000|titleBGColor=#FFF5F5|bgColor=#FFFEF0}}\n" + "{why}\n\n" + "Please resolve this and remove the {{ymir_needs_attention}} flag.\n" + "{{panel}}" +) + +# --- Testing analyst schemas and prompts --- + + +class TestingAnalystInput(BaseModel): + issue: FullIssue = Field(description="Details of JIRA issue to analyze") + maintainer_rules: str = Field( + description="Maintainer-defined rules and guidelines for the package (from AGENTS.md)" + ) + erratum: FullErratum = Field(description="Details of the related ERRATUM") + current_time: datetime = Field(description="Current timestamp") + + +class TestingAnalystOutput(BaseModel): + state: TestingState = Field(description="State of tests") + comment: str | None = Field(description="Comment to add to the JIRA issue") + failed_test_ids: list[str] | None = Field(description="List of Testing Farm run IDs with failures") + + +TESTING_ANALYST_TEMPLATE_COMMON = """\ +You are the testing analyst agent for Project Ymir. Comments that tag +[~jotnar-project] in JIRA issues are directed to you and other Ymir agents +sharing the same account—pay close attention to these. + +Your task is to analyze a RHEL JIRA issue with a fix attached and determine +the state of testing and what needs to be done. + +JIRA_ISSUE_DATA: {{ issue }} +ERRATUM_DATA: {{ erratum }} +MAINTAINER_RULES: {{ maintainer_rules }} +CURRENT_TIME: {{ current_time }} +""" + +TESTING_ANALYST_TEMPLATE_NORMAL = ( + TESTING_ANALYST_TEMPLATE_COMMON + + """ +For components handled by the New Errata Workflow Automation(NEWA): +NEWA will post a comment to the erratum when it has started tests and when they finish. +Read the JIRA issue in those comments to find test results. +For components handled by Errata Workflow Automation (EWA): +EWA will post a comment to the erratum when it has started tests and when they finish. +Read the comment to find the test results in TCMS Test Run. + +If the maintainer rules say that tests are started by NEWA, but there are no comments +from NEWA providing links to JIRA issues, then this component may be a component where +NEWA is only used for RHEL10, and not earlier versions - in that case, you may read the +results from the TCMS test run posted by EWA. + +In all other cases, if the tests are supposed to be started by NEWA, ignore any comments with +links to TCMS or Beaker. + +IMPORTANT: OSCI gating tests run as part of the GitLab merge request pipeline and +they do NOT constitute final testing. You must find evidence of full integration +and regression testing triggered by NEWA or EWA (posted as comments on the erratum) +before concluding tests-passed or tests-waived. If only OSCI gating results are +available, return tests-pending. + +You cannot assume that tests have passed just because a comment says they have +finished, it is mandatory to check the actual test results in the JIRA issue or TCMS. +Make sure that the JIRA issue or TCMS Test Run is the correct one for the latest build in the +erratum. + +Tests can trigger at various points in an issue's lifecycle depending on component +configuration, but always by the time the erratum moves to QE status. If the erratum +is in QE status, and its last_status_transition_timestamp is more than 6 hours ago, +and there's no evidence from erratum comments of tests running or completed, then assume +tests will not run automatically and return tests-not-running. + +Call the final_answer tool passing in the state and a comment as follows. +The comment should use JIRA comment syntax. + +If the tests need to be started manually: + state: tests-not-running + comment: [explain what needs to be done to start tests] + +If the tests are complete and failed: + state: tests-failed + comment: [list failed tests with URLs] + failed_test_ids: [list of IDs for testing farm runs that failed] + +If the tests are complete and passed: + state: tests-passed + comment: [Give a brief summary of what was tested with a link to the result.] + +If there are *some* test failures, but you are sure they are not regressions +and most tests complete successfully: + state: tests-waived + comment: [Explain which tests failed and why they are not considered regressions] + +If the tests will be started automatically without user intervention, but are not yet running: + state: tests-pending + comment: [Provide a brief description of what tests are expected to run and where the results will be] + +If the tests are currently running: + state: tests-running + comment: [Provide a brief description of what tests are running and where the results will be] + +If tests have not started or completed when they should have (as described above): + state: tests-not-running + comment: [Explain the situation and that manual intervention is needed] +""" +) + +TESTING_ANALYST_TEMPLATE_AFTER_BASELINE = ( + TESTING_ANALYST_TEMPLATE_COMMON + + """ +You have previously analyzed this issue and identified failing test runs. These +tests have now been repeated with a baseline build to determine if the failures +are due to issues in the new build, or whether the tests were already failing. + +Please read the comments in the JIRA issue related to find the results of the +baseline test runs, and update your analysis accordingly. The detailed results +of the comparison will be found in the attachments to the JIRA issue. Make +sure to read these attachments for all architectures. + +You can read logfiles to help with your analysis using the read_logfile tool. + +If all tests that failed with the new build also failed with the baseline build, +then it is likely that there are no regressions in the new build. However, +you should examine a selection of log files to make sure that the failures are +consistent between the two runs, and that the failures are not due to some basic +failure of the test environment (e.g. misconfiguration, missing dependencies, +infrastructure issues) that would obscure real regressions. + +An appropriate number of log files to examine in this case is typically 2-3 per +architecture. + +Call the final_answer tool passing in the state and a comment as follows. +The comment should use JIRA comment syntax. If it seems useful, please include +a table in the output comment summarizing the results per architecture. + +If an error prevented tests from running on the new build: + state: tests-error + comment: explanation of why tests could not be run, if available + +If the tests failures seem to reflect a regression in the new build compared to the baseline: + state: tests-failed + comment: detailed description of the tests that are failing, and if known, possible reasons why + +If you are uncertain whether the failures are due to a regression or not: + state: tests-failed + comment: detailed description about what might reflect a regression + +If it seems highly likely that there are no regressions: + state: tests-waived + comment: description of why the failures are not likely to be regressions + + Do not use tests-waived option if tests could not be run on the new build. +""" +) + + +def _render_testing_analyst_prompt(input: TestingAnalystInput, after_baseline: bool) -> str: + template = TESTING_ANALYST_TEMPLATE_AFTER_BASELINE if after_baseline else TESTING_ANALYST_TEMPLATE_NORMAL + return PromptTemplate(PromptTemplateInput(schema=TestingAnalystInput, template=template)).render(input) + + +async def _analyze_testing_results( + jira_issue: FullIssue, + erratum: FullErratum, + gateway_tools: list, + after_baseline: bool = False, +) -> TestingAnalystOutput: + """Run the testing analyst sub-agent to analyze test results.""" + tools = [ + ThinkTool(), + GetJiraAttachmentTool(), + ReadLogfileTool(), + ReadReadmeTool(), + SearchResultsdbTool(), + AnalyzeEwaTestRunTool(), + ] + + agent = ReasoningAgent( + name="TestingAnalyst", + description="Agent that analyzes JIRA issues and determines the state of testing for RHEL errata", + llm=get_chat_model(), + unconstrained=is_reasoning_enabled(), + tool_call_checker=get_tool_call_checker_config(), + tools=tools, + memory=UnconstrainedMemory(), + ) + + maintainer_rules = await run_tool( + "get_maintainer_rules", + available_tools=gateway_tools, + package=jira_issue.components[0], + ) + + input = TestingAnalystInput( + issue=jira_issue, + maintainer_rules=maintainer_rules, + erratum=erratum, + current_time=datetime.now(UTC), + ) + + response = await agent.run( + _render_testing_analyst_prompt(input, after_baseline=after_baseline), + expected_output=TestingAnalystOutput, + **get_agent_execution_config(), # type: ignore + ) + if response.state.result is None: + raise ValueError("Agent did not return a result") + output = TestingAnalystOutput.model_validate_json(response.state.result.text) + logger.info("Testing analysis completed: %s", output.model_dump_json(indent=4)) + return output + + +class IssueVerificationWorkflowState(BaseModel): + jira_issue: str + dry_run: bool = False + ignore_needs_attention: bool = False + + issue_data: dict[str, Any] | None = Field(default=None) + issue: FullIssue | None = Field(default=None) + result: WorkflowResult | None = Field(default=None) + + +def _parse_issue_from_jira_data(issue_data: dict[str, Any]) -> FullIssue: + """Parse a FullIssue from raw JIRA API response data.""" + fields = issue_data.get("fields", {}) + key = issue_data["key"] + jira_url = os.environ.get("JIRA_URL", "https://redhat.atlassian.net").rstrip("/") + + def _get_custom_field(name: str) -> Any: + """Look up a custom field by known name mappings.""" + known_fields = { + "Errata Link": "customfield_10418", + "Fixed in Build": "customfield_10578", + "Test Coverage": "customfield_10638", + "Preliminary Testing": "customfield_10879", + "AssignedTeam": "customfield_10371", + } + field_id = known_fields.get(name) + if field_id: + return fields.get(field_id) + return None + + def _get_enum_value(data: Any) -> str | None: + if isinstance(data, dict): + return data.get("value") + return None + + def _get_enum_list(data: Any) -> list[str] | None: + if data is None: + return None + if isinstance(data, list): + return [d.get("value") for d in data if isinstance(d, dict) and d.get("value")] + return None + + errata_link = _get_custom_field("Errata Link") or fields.get("customfield_10626") + assigned_team = _get_custom_field("AssignedTeam") + assigned_team_name = assigned_team.get("value") if isinstance(assigned_team, dict) else None + + test_coverage_raw = _get_custom_field("Test Coverage") + test_coverage = None + if test_coverage_raw: + tc_values = _get_enum_list(test_coverage_raw) + if tc_values: + test_coverage = [TestCoverage(v) for v in tc_values] + + preliminary_testing_raw = _get_custom_field("Preliminary Testing") + preliminary_testing = None + if preliminary_testing_raw: + pt_value = _get_enum_value(preliminary_testing_raw) + if pt_value: + preliminary_testing = PreliminaryTesting(pt_value) + + # Extract description - handle both plain text (v2) and ADF (v3) formats + description_raw = fields.get("description", "") + if isinstance(description_raw, dict): + # ADF format - extract text content + def _extract_adf_text(node: Any) -> str: + if isinstance(node, str): + return node + if isinstance(node, dict): + if node.get("type") == "text": + return node.get("text", "") + content = node.get("content", []) + return "".join(_extract_adf_text(c) for c in content) + if isinstance(node, list): + return "".join(_extract_adf_text(c) for c in node) + return "" + + description = _extract_adf_text(description_raw) + else: + description = description_raw or "" + + # Parse comments + comments_data = fields.get("comment", {}) + if isinstance(comments_data, dict): + comments_list = comments_data.get("comments", []) + elif isinstance(comments_data, list): + comments_list = comments_data + else: + comments_list = [] + + comments = [ + JiraComment( + authorName=c["author"].get("displayName", "Unknown"), + authorEmail=c["author"].get("emailAddress"), + created=datetime.fromisoformat(c["created"]), + body=c.get("body", ""), + id=c["id"], + ) + for c in comments_list + ] + + return FullIssue( + key=key, + url=f"{jira_url}/browse/{key}", + assigned_team=assigned_team_name, + summary=fields.get("summary", ""), + status=IssueStatus(fields.get("status", {}).get("name", "New")), + components=[c["name"] for c in fields.get("components", [])], + labels=fields.get("labels", []), + fix_versions=[v["name"] for v in fields.get("fixVersions", [])], + errata_link=errata_link, + fixed_in_build=_get_custom_field("Fixed in Build"), + test_coverage=test_coverage, + preliminary_testing=preliminary_testing, + description=description, + comments=comments, + ) + + +async def _add_label(issue_key: str, label: str, comment: str | None, tools: list, dry_run: bool) -> None: + if dry_run: + logger.info("Dry run: would add label %s to issue %s", label, issue_key) + return + + await run_tool( + "edit_jira_labels", + available_tools=tools, + issue_key=issue_key, + labels_to_add=[label], + ) + if comment: + await run_tool( + "add_jira_comment", + available_tools=tools, + issue_key=issue_key, + comment=comment, + private=True, + ) + + +async def _remove_label(issue_key: str, label: str, tools: list, dry_run: bool) -> None: + if dry_run: + logger.info("Dry run: would remove label %s from issue %s", label, issue_key) + return + + await run_tool( + "edit_jira_labels", + available_tools=tools, + issue_key=issue_key, + labels_to_remove=[label], + ) + + +async def _flag_attention( + issue_key: str, + why: str, + *, + details_comment: str | None = None, + tools: list, + dry_run: bool, +) -> WorkflowResult: + full_comment = ATTENTION_TEMPLATE.format(why=why) + if details_comment: + full_comment = f"{full_comment}\n\n{details_comment}" + + await _add_label(issue_key, JiraLabels.NEEDS_ATTENTION.value, full_comment, tools, dry_run) + return WorkflowResult(status=why, reschedule_in=-1) + + +async def _change_status( + issue_key: str, + current_status: IssueStatus, + new_status: IssueStatus, + why: str, + tools: list, + dry_run: bool, +) -> WorkflowResult: + comment = f"*Changing status from {current_status} => {new_status}*\n\n{why}" + + if dry_run: + logger.info("Dry run: would change issue %s status to %s", issue_key, new_status) + else: + await run_tool( + "change_jira_status", + available_tools=tools, + issue_key=issue_key, + status=str(new_status), + ) + await run_tool( + "add_jira_comment", + available_tools=tools, + issue_key=issue_key, + comment=comment, + private=True, + ) + + reschedule_delay = -1 if new_status in (IssueStatus.RELEASE_PENDING, IssueStatus.CLOSED) else 0 + return WorkflowResult(status=why, reschedule_in=reschedule_delay) + + +async def _search_merged_mrs( + component: str, issue_key: str, state: str, tools: list +) -> list[GitlabMergeRequest]: + """Search for merge requests across all GitLab groups.""" + results = [] + for group in GITLAB_GROUPS: + project = f"redhat/{group}/{component}" + try: + mrs_data = await run_tool( + "search_gitlab_project_mrs", + available_tools=tools, + project=project, + search=issue_key, + state=state, + ) + results.extend(GitlabMergeRequest(**mr_data) for mr_data in mrs_data if isinstance(mr_data, dict)) + except Exception as e: + logger.warning("Error searching MRs in %s: %s", project, e) + return results + + +async def _label_merge_if_needed(issue: FullIssue, tools: list, dry_run: bool) -> bool: + """Add ymir_merged label if a merged MR exists and the issue has backported/rebased label.""" + component = issue.components[0] + + if ( + JiraLabels.BACKPORTED.value in issue.labels or JiraLabels.REBASED.value in issue.labels + ) and JiraLabels.MERGED.value not in issue.labels: + merged_mrs = await _search_merged_mrs(component, issue.key, GitlabMergeRequestState.MERGED, tools) + if merged_mrs: + merged_mr = merged_mrs[0] + await _add_label( + issue.key, + JiraLabels.MERGED.value, + f"A [merge request|{merged_mr.url}]. resolving this issue " + "has been merged; waiting for errata creation and final testing.", + tools, + dry_run, + ) + issue.labels.append(JiraLabels.MERGED.value) + return True + + return False + + +async def _get_latest_merged_timestamp(issue: FullIssue, tools: list) -> datetime: + """Get the latest merged timestamp from all merged MRs.""" + component = issue.components[0] + merged_mrs = await _search_merged_mrs(component, issue.key, GitlabMergeRequestState.MERGED, tools) + if not merged_mrs: + return DATETIME_MIN_UTC + return max( + (mr.merged_at or DATETIME_MIN_UTC for mr in merged_mrs), + default=DATETIME_MIN_UTC, + ) + + +async def run_issue_verification( + jira_issue: str, + dry_run: bool = False, + ignore_needs_attention: bool = False, +) -> WorkflowResult: + async with mcp_tools(os.getenv("MCP_GATEWAY_URL")) as gateway_tools: + workflow = Workflow(IssueVerificationWorkflowState, name="IssueVerificationWorkflow") + + async def fetch_and_validate_issue(state: IssueVerificationWorkflowState): + """Fetch JIRA issue data and validate preconditions.""" + logger.info("Fetching JIRA issue data for %s", state.jira_issue) + state.issue_data = await run_tool( + "get_jira_details", + available_tools=gateway_tools, + issue_key=state.jira_issue, + ) + + state.issue = _parse_issue_from_jira_data(state.issue_data) + issue = state.issue + + logger.info("Running workflow for issue %s", issue.url) + + if JiraLabels.NEEDS_ATTENTION.value in issue.labels and not state.ignore_needs_attention: + state.result = WorkflowResult( + status="Issue has the ymir_needs_attention label", + reschedule_in=-1, + ) + return Workflow.END + + if len(issue.components) != 1: + state.result = await _flag_attention( + issue.key, + "This issue has multiple components. " + "Ymir only handles issues with single component currently.", + tools=gateway_tools, + dry_run=state.dry_run, + ) + return Workflow.END + + return "check_errata_status" + + async def check_errata_status(state: IssueVerificationWorkflowState): + """Branch based on whether errata link exists.""" + if state.issue.errata_link is None: + return "run_before_errata" + return "run_after_errata" + + async def run_before_errata(state: IssueVerificationWorkflowState): + """Handle issues without errata link.""" + issue = state.issue + + if not any( + label + in ( + JiraLabels.BACKPORTED.value, + JiraLabels.REBASED.value, + JiraLabels.MERGED.value, + ) + for label in issue.labels + ): + state.result = WorkflowResult( + status=f"Issue without target labels: {issue.labels}", + reschedule_in=-1, + ) + return Workflow.END + + if JiraLabels.MERGED.value not in issue.labels: + await _label_merge_if_needed(issue, gateway_tools, state.dry_run) + + if JiraLabels.MERGED.value not in issue.labels: + state.result = WorkflowResult( + status=f"No merged MR found, reschedule in {MERGE_CHECK_DELAY}s", + reschedule_in=MERGE_CHECK_DELAY, + ) + return Workflow.END + + latest_merged_timestamp = await _get_latest_merged_timestamp(issue, gateway_tools) + cur_time = datetime.now(tz=UTC) + time_diff = abs(cur_time - latest_merged_timestamp) + if time_diff < timedelta(days=1): + state.result = WorkflowResult( + status=f"Wait for the associated erratum to be created, " + f"reschedule in {ERRATA_WAIT_DELAY}s", + reschedule_in=ERRATA_WAIT_DELAY, + ) + return Workflow.END + + state.result = await _flag_attention( + issue.key, + "A merge request was merged for this issue more than 24 hours ago but no errata " + "was created. Please investigate and look for gating failures or other reasons " + "that might have blocked errata creation.", + tools=gateway_tools, + dry_run=state.dry_run, + ) + return Workflow.END + + async def run_after_errata(state: IssueVerificationWorkflowState): + """Handle issues with errata link.""" + issue = state.issue + assert issue.errata_link is not None + + if issue.fixed_in_build is None: + state.result = await _flag_attention( + issue.key, + "Issue has errata_link but no fixed_in_build", + tools=gateway_tools, + dry_run=state.dry_run, + ) + return Workflow.END + + if issue.preliminary_testing != PreliminaryTesting.PASS: + state.result = await _flag_attention( + issue.key, + "Issue does not have Preliminary Testing set to Pass - this should have " + "happened before the gitlab pull request was merged", + tools=gateway_tools, + dry_run=state.dry_run, + ) + return Workflow.END + + if issue.test_coverage is None or len(issue.test_coverage) == 0: + state.result = await _flag_attention( + issue.key, + "Issue does not have Test Coverage set - this should have " + "happened before the gitlab pull request was merged", + tools=gateway_tools, + dry_run=state.dry_run, + ) + return Workflow.END + + # Add merged label even in post-errata state for JIRA dashboards + await _label_merge_if_needed(issue, gateway_tools, state.dry_run) + + match issue.status: + case IssueStatus.NEW | IssueStatus.PLANNING | IssueStatus.IN_PROGRESS: + state.result = await _change_status( + issue.key, + issue.status, + IssueStatus.INTEGRATION, + "Preliminary testing has passed, moving to Integration", + gateway_tools, + state.dry_run, + ) + return Workflow.END + case IssueStatus.INTEGRATION: + if "ymir_reproducing_tests" in issue.labels: + return "check_reproduction" + return "analyze_testing" + case IssueStatus.RELEASE_PENDING | IssueStatus.CLOSED: + state.result = WorkflowResult( + status=f"Issue status is {issue.status}", + reschedule_in=-1, + ) + return Workflow.END + case _: + raise ValueError(f"Unknown issue status: {issue.status}") + + async def analyze_testing(state: IssueVerificationWorkflowState): + """Call testing_analyst sub-agent for test result analysis.""" + issue = state.issue + assert issue.errata_link is not None + + # Get erratum data + erratum_data = await run_tool( + "get_erratum", + available_tools=gateway_tools, + erratum_id=issue.errata_link, + full=True, + ) + related_erratum = FullErratum(**erratum_data) + + # Check if baseline tests were already run + baseline_tests = BaselineTests.load_from_issue(issue) + after_baseline = baseline_tests is not None + + testing_analysis = await _analyze_testing_results( + issue, related_erratum, gateway_tools, after_baseline=after_baseline + ) + + match testing_analysis.state: + case TestingState.NOT_RUNNING: + state.result = await _flag_attention( + issue.key, + "Tests aren't running - see details below", + details_comment=testing_analysis.comment, + tools=gateway_tools, + dry_run=state.dry_run, + ) + case TestingState.PENDING: + state.result = WorkflowResult( + status="Tests are pending", + reschedule_in=WAIT_DELAY, + ) + case TestingState.RUNNING: + state.result = WorkflowResult( + status="Tests are running", + reschedule_in=WAIT_DELAY, + ) + case TestingState.FAILED: + if testing_analysis.failed_test_ids and ( + baseline_tests is None + or ( + set(testing_analysis.failed_test_ids) + != {c.failed.id for c in baseline_tests.comparisons} + ) + ): + # Start reproduction with baseline build + previous_build_nvr = await run_tool( + "get_erratum_build_nvr", + available_tools=gateway_tools, + erratum_id=related_erratum.id, + component=issue.components[0], + ) + + if previous_build_nvr is None: + state.result = await _flag_attention( + issue.key, + "Tests failed - see details below. " + "Cannot start reproduction with previous build " + "- error finding previous build NVR.", + details_comment=testing_analysis.comment, + tools=gateway_tools, + dry_run=state.dry_run, + ) + else: + try: + new_baseline_tests = await BaselineTests.create( + failure_comment=testing_analysis.comment or "", + failed_request_ids=testing_analysis.failed_test_ids, + previous_build_nvr=previous_build_nvr, + dry_run=state.dry_run, + tools=gateway_tools, + ) + issue_comment = await new_baseline_tests.format_issue_comment( + tools=gateway_tools + ) + await _add_label( + issue.key, + "ymir_reproducing_tests", + issue_comment, + gateway_tools, + state.dry_run, + ) + state.result = WorkflowResult( + status="Waiting to reproduce tests with previous build", + reschedule_in=WAIT_DELAY, + ) + except Exception as e: + logger.exception("Failed to start test reproduction: %s", e) + state.result = await _flag_attention( + issue.key, + f"Tests failed - see details below. {e}", + details_comment=testing_analysis.comment, + tools=gateway_tools, + dry_run=state.dry_run, + ) + else: + state.result = await _flag_attention( + issue.key, + "Tests failed - see details below", + details_comment=testing_analysis.comment, + tools=gateway_tools, + dry_run=state.dry_run, + ) + case TestingState.ERROR: + state.result = await _flag_attention( + issue.key, + "An error occurred during testing - see details below", + details_comment=testing_analysis.comment, + tools=gateway_tools, + dry_run=state.dry_run, + ) + case TestingState.PASSED: + state.result = await _change_status( + issue.key, + issue.status, + IssueStatus.RELEASE_PENDING, + testing_analysis.comment or "Final testing has passed.", + gateway_tools, + state.dry_run, + ) + case TestingState.WAIVED: + state.result = await _change_status( + issue.key, + issue.status, + IssueStatus.RELEASE_PENDING, + testing_analysis.comment + or "Final testing has been waived, moving to Release Pending.", + gateway_tools, + state.dry_run, + ) + case _: + raise ValueError(f"Unknown testing state: {testing_analysis.state}") + + return Workflow.END + + async def check_reproduction(state: IssueVerificationWorkflowState): + """Check status of baseline test reproduction.""" + issue = state.issue + baseline_tests = BaselineTests.load_from_issue(issue) + + if baseline_tests is None: + state.result = await _flag_attention( + issue.key, + "Issue has ymir_reproducing_tests label but cannot parse baseline tests from comments", + tools=gateway_tools, + dry_run=state.dry_run, + ) + return Workflow.END + + if not await baseline_tests.settled(gateway_tools): + state.result = WorkflowResult( + status="Waiting for baseline tests to complete", + reschedule_in=WAIT_DELAY, + ) + return Workflow.END + + await baseline_tests.create_attachments( + issue_key=issue.key, + dry_run=state.dry_run, + tools=gateway_tools, + ) + + issue_comment = await baseline_tests.format_issue_comment( + include_attachments=True, tools=gateway_tools + ) + await _remove_label(issue.key, "ymir_reproducing_tests", gateway_tools, state.dry_run) + + # Update the existing comment + assert baseline_tests.comment_id is not None + if not state.dry_run: + await run_tool( + "update_jira_comment", + available_tools=gateway_tools, + issue_key=issue.key, + comment_id=baseline_tests.comment_id, + comment=issue_comment, + ) + + state.result = WorkflowResult( + status="Baseline tests are complete, will analyze results", + reschedule_in=0.0, + ) + return Workflow.END + + workflow.add_step("fetch_and_validate_issue", fetch_and_validate_issue) + workflow.add_step("check_errata_status", check_errata_status) + workflow.add_step("run_before_errata", run_before_errata) + workflow.add_step("run_after_errata", run_after_errata) + workflow.add_step("analyze_testing", analyze_testing) + workflow.add_step("check_reproduction", check_reproduction) + + response = await workflow.run( + IssueVerificationWorkflowState( + jira_issue=jira_issue, + dry_run=dry_run, + ignore_needs_attention=ignore_needs_attention, + ) + ) + + return response.state.result + + +async def main() -> None: + logging.basicConfig(level=logging.INFO) + + setup_observability(os.environ["COLLECTOR_ENDPOINT"]) + + dry_run = os.getenv("DRY_RUN", "False").lower() == "true" + ignore_needs_attention = os.getenv("IGNORE_NEEDS_ATTENTION", "false").lower() == "true" + + jira_issue = os.getenv("JIRA_ISSUE") + if not jira_issue: + logger.error("JIRA_ISSUE environment variable is required") + sys.exit(1) + + logger.info("Running issue verification for %s (dry_run=%s)", jira_issue, dry_run) + result = await run_issue_verification( + jira_issue, + dry_run=dry_run, + ignore_needs_attention=ignore_needs_attention, + ) + logger.info("Completed: status=%s, reschedule_in=%s", result.status, result.reschedule_in) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except FrameworkError as e: + traceback.print_exc() + sys.exit(e.explain()) diff --git a/ymir/agents/preliminary_testing_agent.py b/ymir/agents/preliminary_testing_agent.py index 6e344ca2..1621a911 100644 --- a/ymir/agents/preliminary_testing_agent.py +++ b/ymir/agents/preliminary_testing_agent.py @@ -295,11 +295,12 @@ async def gather_test_sources(state: PreliminaryTestingWorkflowState): state.build_nvr = fields.get(FIXED_IN_BUILD_CUSTOM_FIELD) try: - state.pull_requests = await run_tool( + pr_result = await run_tool( "get_jira_pull_requests", available_tools=gateway_tools, issue_key=state.jira_issue, ) + state.pull_requests = pr_result.get("pull_requests", []) except Exception as e: logger.warning("Failed to get pull requests for %s: %s", state.jira_issue, e) diff --git a/ymir/agents/tests/unit/test_zstream_search.py b/ymir/agents/tests/unit/test_zstream_search.py index 8e936367..23159b1b 100644 --- a/ymir/agents/tests/unit/test_zstream_search.py +++ b/ymir/agents/tests/unit/test_zstream_search.py @@ -173,13 +173,15 @@ async def test_found_in_closest_stream(): }, }, ] - dev_status_result = [ - { - "url": "https://gitlab.com/redhat/rhel/rpms/fence-agents/-/commit/abc123", - "message": "fix missing statuses", - "repository_url": "https://gitlab.com/redhat/rhel/rpms/fence-agents", - }, - ] + dev_status_result = { + "commits": [ + { + "url": "https://gitlab.com/redhat/rhel/rpms/fence-agents/-/commit/abc123", + "message": "fix missing statuses", + "repository_url": "https://gitlab.com/redhat/rhel/rpms/fence-agents", + }, + ] + } mock_run_tool = AsyncMock(side_effect=[search_result, dev_status_result]) @@ -226,14 +228,16 @@ async def test_cascade_to_further_version(): }, }, ] - dev_status_empty = [] - dev_status_with_commits = [ - { - "url": "https://gitlab.com/redhat/centos-stream/rpms/fence-agents/-/commit/def456", - "message": "fix issue", - "repository_url": "https://gitlab.com/redhat/centos-stream/rpms/fence-agents", - }, - ] + dev_status_empty = {"commits": []} + dev_status_with_commits = { + "commits": [ + { + "url": "https://gitlab.com/redhat/centos-stream/rpms/fence-agents/-/commit/def456", + "message": "fix issue", + "repository_url": "https://gitlab.com/redhat/centos-stream/rpms/fence-agents", + }, + ] + } mock_run_tool = AsyncMock(side_effect=[search_result, dev_status_empty, dev_status_with_commits]) @@ -271,7 +275,7 @@ async def test_not_found_anywhere(): }, }, ] - dev_status_empty = [] + dev_status_empty = {"commits": []} mock_run_tool = AsyncMock(side_effect=[search_result, dev_status_empty]) @@ -342,13 +346,15 @@ async def test_version_proximity_sorting(): }, ] # The closer issue (9.7.z) should be tried first and has commits - dev_status_with_commits = [ - { - "url": "https://gitlab.com/repo/-/commit/abc", - "message": "fix", - "repository_url": "https://gitlab.com/repo", - }, - ] + dev_status_with_commits = { + "commits": [ + { + "url": "https://gitlab.com/repo/-/commit/abc", + "message": "fix", + "repository_url": "https://gitlab.com/repo", + }, + ] + } mock_run_tool = AsyncMock(side_effect=[search_result, dev_status_with_commits]) diff --git a/ymir/agents/utilities/__init__.py b/ymir/agents/utilities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ymir/agents/utilities/baseline_tests.py b/ymir/agents/utilities/baseline_tests.py new file mode 100644 index 00000000..b89b7fac --- /dev/null +++ b/ymir/agents/utilities/baseline_tests.py @@ -0,0 +1,329 @@ +import logging +import re +from dataclasses import dataclass +from functools import cached_property + +from ymir.agents.utilities.compare_xunit import ( + XUnitComparison, + XUnitComparisonStatus, + compare_xunit_files, +) +from ymir.common.models import ( + FullIssue, + TestingFarmRequest, + TestingFarmRequestState, +) +from ymir.common.utils import run_tool + +logger = logging.getLogger(__name__) + +TESTING_FARM_URL = "https://api.testing-farm.io" + + +class RequestWrapper: + """ + Wrapper around TestingFarmRequest or request ID string + + We start off with either a TestingFarmRequest object or a string ID, and + lazily fetch the TestingFarmRequest object only when needed. This + also has convenience properties for ID, URL, and JIRA-formatted link. + """ + + def __init__(self, request: str | TestingFarmRequest): + self._request = request + + async def async_request(self, tools: list) -> TestingFarmRequest: + if isinstance(self._request, TestingFarmRequest): + return self._request + data = await run_tool( + "get_testing_farm_request", + available_tools=tools, + request_id=self._request, + ) + self._request = TestingFarmRequest(**data) + return self._request + + @cached_property + def request(self) -> TestingFarmRequest: + if isinstance(self._request, TestingFarmRequest): + return self._request + raise RuntimeError("Request not yet fetched; call async_request() first") + + @property + def id(self) -> str: + if isinstance(self._request, TestingFarmRequest): + return self._request.id + return self._request + + @property + def url(self) -> str: + return f"{TESTING_FARM_URL}/requests/{self.id}" + + @property + def link(self) -> str: + return f"[{self.id}|{self.url}]" + + +class BaselineComparison: + def __init__(self, failed: str | TestingFarmRequest, baseline: str | TestingFarmRequest): + self.failed = RequestWrapper(failed) + self.baseline = RequestWrapper(baseline) + + @property + def attachment_name(self) -> str: + return f"comparison-{self.baseline.id}--{self.failed.id}.toml" + + @property + def attachment_link(self) -> str: + return f"[compare|^{self.attachment_name}]" + + +@dataclass(kw_only=True) +class BaselineTests: + failure_comment: str + comparisons: list[BaselineComparison] + previous_build_nvr: str + comment_id: str | None = None + + async def settled(self, tools: list) -> bool: + for comparison in self.comparisons: + request = await comparison.baseline.async_request(tools) + if request.state not in ( + TestingFarmRequestState.COMPLETE, + TestingFarmRequestState.ERROR, + TestingFarmRequestState.CANCELED, + ): + return False + return True + + async def complete(self, tools: list) -> bool: + for comparison in self.comparisons: + request = await comparison.baseline.async_request(tools) + if request.state != TestingFarmRequestState.COMPLETE: + return False + return True + + @staticmethod + def _request_outcome(request: TestingFarmRequest) -> str: + if request.state == TestingFarmRequestState.COMPLETE: + return request.result + return request.state + + async def format_issue_comment( + self, *, include_attachments: bool = False, tools: list | None = None + ) -> str: + if tools: + is_complete = await self.complete(tools) + is_settled = await self.settled(tools) + else: + # Fallback for when requests are already loaded + is_complete = all( + isinstance(c.baseline._request, TestingFarmRequest) + and c.baseline._request.state == TestingFarmRequestState.COMPLETE + for c in self.comparisons + ) + is_settled = all( + isinstance(c.baseline._request, TestingFarmRequest) + and c.baseline._request.state + in ( + TestingFarmRequestState.COMPLETE, + TestingFarmRequestState.ERROR, + TestingFarmRequestState.CANCELED, + ) + for c in self.comparisons + ) + + if is_complete: + message = "Reproduced" + state_header = "Result" + elif is_settled: + message = "Failed to reproduce" + state_header = "Result" + else: + message = "Reproducing" + state_header = "State" + + return ( + self.failure_comment + + "\n\n" + + ( + f"{message} failed tests with previous build {self.previous_build_nvr}:\n" + f"||Architecture||Original Request||Request With Old Build||{state_header}" + f"{'||Comparison' if include_attachments else ''}" + "||\n" + + "\n".join( + f"|{', '.join(comparison.failed.request.arches)}" + f"|{comparison.failed.link}" + f"|{comparison.baseline.link}" + f"|{self._request_outcome(comparison.baseline.request)}" + f"{('|' + comparison.attachment_link) if include_attachments else ''}" + f"|" + for comparison in self.comparisons + ) + ) + ) + + async def create_attachments( + self, issue_key: str, dry_run: bool = False, tools: list | None = None + ) -> None: + attachments: list[tuple[str, bytes, str]] = [] + for comparison in self.comparisons: + failed_request = comparison.failed.request + baseline_request = comparison.baseline.request + + metadata = {} + + metadata |= { + "build_a": baseline_request.build_nvr, + "testing_farm_request_id_a": baseline_request.id, + } + if baseline_request.error_reason: + metadata["error_reason_a"] = baseline_request.error_reason + + metadata |= { + "build_b": failed_request.build_nvr, + "testing_farm_request_id_b": failed_request.id, + } + if failed_request.error_reason: + metadata["error_reason_b"] = failed_request.error_reason + + def create_not_generated_comparison(reason: str, metadata=metadata) -> XUnitComparison: + return XUnitComparison( + status=XUnitComparisonStatus( + generated=False, + reason=reason, + ), + metadata=metadata, + ) + + match (baseline_request.result_xunit_url, failed_request.result_xunit_url): + case (None, None): + comparison_result = create_not_generated_comparison( + "XUnit results missing for runs A and B" + ) + case (None, _): + comparison_result = create_not_generated_comparison("XUnit results missing for run A") + case (_, None): + comparison_result = create_not_generated_comparison("XUnit results missing for run B") + case _: + comparison_result = await compare_xunit_files( + baseline_request.result_xunit_url, + failed_request.result_xunit_url, + metadata=metadata, + ) + + attachment_bytes = comparison_result.to_toml().encode("utf-8") + logger.info("About to attach %s", attachment_bytes.decode("utf-8")) + attachments.append((comparison.attachment_name, attachment_bytes, "text/plain")) + + if tools: + await run_tool( + "add_jira_attachments", + available_tools=tools, + issue_key=issue_key, + attachments=[ + {"filename": name, "content": content.decode("utf-8")} for name, content, _ in attachments + ], + ) + else: + logger.warning("No tools available, cannot upload attachments for %s", issue_key) + + @staticmethod + async def create( + failure_comment: str, + failed_request_ids: list[str], + previous_build_nvr: str, + dry_run: bool = False, + tools: list | None = None, + ) -> "BaselineTests": + tests: list[BaselineComparison] = [] + + for failed_request_id in failed_request_ids: + if tools: + failed_data = await run_tool( + "get_testing_farm_request", + available_tools=tools, + request_id=failed_request_id, + ) + failed_request = TestingFarmRequest(**failed_data) + else: + raise RuntimeError("Tools required for creating baseline tests") + + logger.info( + "Starting reproduction with previous build %s for failed test run %s", + previous_build_nvr, + failed_request.id, + ) + + try: + if tools: + baseline_data = await run_tool( + "reproduce_testing_farm_request", + available_tools=tools, + request_id=failed_request.id, + build_nvr=previous_build_nvr, + ) + baseline_request = TestingFarmRequest(**baseline_data) + else: + raise RuntimeError("Tools required for creating baseline tests") + + tests.append(BaselineComparison(failed=failed_request, baseline=baseline_request)) + except Exception as e: + raise RuntimeError( + f"Failed to start reproduction of test run {failed_request.id} " + f"with previous build {previous_build_nvr}: {e}", + ) from e + + return BaselineTests( + failure_comment=failure_comment, + comparisons=tests, + previous_build_nvr=previous_build_nvr, + ) + + @staticmethod + def load_from_issue(issue: FullIssue) -> "BaselineTests | None": + for comment in reversed(issue.comments): + lines = comment.body.splitlines() + leading_lines = [] + + line_iter = iter(lines) + for line in line_iter: + if m := re.match( + ".*failed tests with previous build (.*):", + line, + ): + previous_build_nvr = m.group(1) + break + + leading_lines.append(line) + else: + continue + + for line in line_iter: + if line.startswith("||"): + break + else: + continue + + comparisons: list[BaselineComparison] = [] + for line in line_iter: + if not line.startswith("|"): + break + + line = re.sub(r"\[([^|]+)\|([^]]+)\]", r"\1", line) # Remove links + parts = line.split("|") + # parts should be ["", "", "", "", ...] + if len(parts) >= 4: + failed_request_id = parts[2].strip() + baseline_request_id = parts[3].strip() + comparisons.append( + BaselineComparison(failed=failed_request_id, baseline=baseline_request_id) + ) + return BaselineTests( + failure_comment="\n".join(leading_lines).strip(), + comparisons=comparisons, + previous_build_nvr=previous_build_nvr, + comment_id=comment.id, + ) + + return None diff --git a/ymir/agents/utilities/compare_xunit.py b/ymir/agents/utilities/compare_xunit.py new file mode 100644 index 00000000..79fd6393 --- /dev/null +++ b/ymir/agents/utilities/compare_xunit.py @@ -0,0 +1,315 @@ +import asyncio +import xml.etree.ElementTree as ET +from enum import StrEnum + +import aiohttp +import tomli_w +from pydantic import BaseModel, Field + + +class XUnitComparisonStatus(BaseModel): + generated: bool + reason: str | None = None + + +class XUnitTestCaseResult(StrEnum): + PASS = "pass" + FAIL = "fail" + ERROR = "error" + SKIPPED = "skipped" + MISSING = "missing" + + +class XUnitTestCase(BaseModel): + name: str + url: str + ref: str + log_url: str + result: XUnitTestCaseResult + + +class XUnitTestSuite(BaseModel): + arch: str + name: str + test_cases: list[XUnitTestCase] + + +class ComparisonResult(StrEnum): + WORKS = "works" + """Test case that passed in both reports.""" + REGRESSION = "regression" + """Test case that passed in the first report but failed or errored in the second.""" + FIXED = "fixed" + """Test case that failed or errored in the first report but passed in the second.""" + BROKEN = "broken" + """Test case that failed or errored in both reports.""" + DIFFERENCE = "difference" + """Test case with other combinations of status.""" + + +class XUnitTestCaseComparison(BaseModel): + name: str + arch: str + url: str + ref: str + result_a: XUnitTestCaseResult + result_b: XUnitTestCaseResult + log_url_a: str + log_url_b: str + + +class XUnitComparisonCounts(BaseModel): + works: int = 0 + regression: int = 0 + fixed: int = 0 + broken: int = 0 + difference: int = 0 + + +TOML_HEADER = """\ +# XUnit Comparison Report +# It contains the results of comparing two XUnit test reports. +# Each section lists test cases that differ between the two reports. +# The 'total_counts' section summarizes the number of test cases in each comparison category. +# +# Comparison categories: +# - regression: Test cases that passed in the first report but failed in the second. +# - fixed: Test cases that failed in the first report but passed in the second. +# - broken: Test cases that failed in both reports. +# - works: Test cases that passed in both reports (not listed in detail). +# - difference: Test cases with other combinations of status. +""" + + +class XUnitComparison(BaseModel): + status: XUnitComparisonStatus + metadata: dict[str, str] = Field(default_factory=dict) + total_counts: XUnitComparisonCounts = Field(default_factory=XUnitComparisonCounts) + regression: list[XUnitTestCaseComparison] = Field(default_factory=list) + fixed: list[XUnitTestCaseComparison] = Field(default_factory=list) + broken: list[XUnitTestCaseComparison] = Field(default_factory=list) + difference: list[XUnitTestCaseComparison] = Field(default_factory=list) + + def to_toml(self) -> str: + dict_output = self.model_dump() + + # Clean up the output + if sum(c for c in dict_output["total_counts"].values()) == 0: + del dict_output["total_counts"] + + empty_lists = [k for k, v in dict_output.items() if isinstance(v, list) and not v] + for k in empty_lists: + del dict_output[k] + + return TOML_HEADER + tomli_w.dumps(dict_output) + + +class XUnitParseError(Exception): + pass + + +def parse_xunit(xml_content: str) -> list[XUnitTestSuite]: + try: + root = ET.fromstring(xml_content) + except ET.ParseError as e: + raise XUnitParseError("Failed to parse XUnit XML") from e + + test_suites = [] + + for testsuite_el in root.findall("./testsuite"): + test_cases = [] + + arch_el = testsuite_el.find("./testing-environment[@name='provisioned']/property[@name='arch']") + arch = arch_el.get("value") if arch_el is not None else None + if arch is None: + raise ValueError("Architecture not found in XUnit XML") + + for testcase_el in testsuite_el.findall("./testcase"): + name = testcase_el.get("name") + if not name: + raise XUnitParseError("Test case without name found") + + fmf_id_el = testcase_el.find("fmf-id") + if fmf_id_el is None: + raise XUnitParseError(f"Test case {name} doesn't have fmf-id") + url = fmf_id_el.get("url") + ref = fmf_id_el.get("ref") + + if not url or not ref: + raise XUnitParseError(f"Test case {name} has incomplete fmf-id") + + log_el = testcase_el.find("./logs/log[@name='testout.log']") + log_url = log_el.get("href") if log_el is not None else None + + if log_url is None: + log_url = "" + + if testcase_el.find("failure") is not None: + result = XUnitTestCaseResult.FAIL + elif testcase_el.find("error") is not None: + result = XUnitTestCaseResult.ERROR + elif testcase_el.find("skipped") is not None: + result = XUnitTestCaseResult.SKIPPED + else: + result = XUnitTestCaseResult.PASS + + test_case = XUnitTestCase( + name=name, + url=url, + ref=ref, + log_url=log_url, + result=result, + ) + test_cases.append(test_case) + + test_suite = XUnitTestSuite( + name=testsuite_el.get("name") or "unknown", + arch=arch, + test_cases=test_cases, + ) + test_suites.append(test_suite) + + return test_suites + + +def compare_test_suites(suite_a: XUnitTestSuite, suite_b: XUnitTestSuite, output: XUnitComparison): + """Compare two test suites and update the output. + + Differences *from* suite_a *to* suite_b are recorded in the output. + """ + + case_map_a = {tc.name: tc for tc in suite_a.test_cases} + case_map_b = {tc.name: tc for tc in suite_b.test_cases} + + all_keys = set(case_map_a.keys()) | set(case_map_b.keys()) + + for key in all_keys: + tc_a = case_map_a.get(key) + tc_b = case_map_b.get(key) + + some_case = tc_a or tc_b + assert some_case is not None + + result_a = tc_a.result if tc_a else XUnitTestCaseResult.MISSING + result_b = tc_b.result if tc_b else XUnitTestCaseResult.MISSING + + log_url_a = tc_a.log_url if tc_a else "" + log_url_b = tc_b.log_url if tc_b else "" + + if result_a == XUnitTestCaseResult.PASS and result_b == XUnitTestCaseResult.PASS: + comparison_result = ComparisonResult.WORKS + elif result_a == XUnitTestCaseResult.PASS and result_b in ( + XUnitTestCaseResult.FAIL, + XUnitTestCaseResult.ERROR, + ): + comparison_result = ComparisonResult.REGRESSION + elif ( + result_a in (XUnitTestCaseResult.FAIL, XUnitTestCaseResult.ERROR) + and result_b == XUnitTestCaseResult.PASS + ): + comparison_result = ComparisonResult.FIXED + elif result_a in ( + XUnitTestCaseResult.FAIL, + XUnitTestCaseResult.ERROR, + ) and result_b in (XUnitTestCaseResult.FAIL, XUnitTestCaseResult.ERROR): + comparison_result = ComparisonResult.BROKEN + else: + comparison_result = ComparisonResult.DIFFERENCE + + old_count = getattr(output.total_counts, comparison_result.value, 0) + setattr(output.total_counts, comparison_result.value, old_count + 1) + + if comparison_result != ComparisonResult.WORKS: + comparison = XUnitTestCaseComparison( + name=some_case.name, + arch=suite_a.arch, + url=some_case.url, + ref=some_case.ref, + result_a=result_a, + result_b=result_b, + log_url_a=log_url_a, + log_url_b=log_url_b, + ) + + getattr(output, comparison_result.value).append(comparison) + + +async def compare_xunit_files( + xunit_url_a: str, xunit_url_b: str, *, metadata: dict[str, str] | None = None +) -> XUnitComparison: + """ + Download and compare two XUnit files. + + Downloads XUnit files from the given URLs, compare them, and returns a + comparison result that includes counts of different comparison results and + details about test cases that differ. + + Note that the format of the files is Testing Farm's version of the JUnit + XML format. While this is sometimes referred to as the xUnit format and + is widely used across different testing frameworks, it is entirely + different than the format from xUnit.net. "XUnit" is used here + only for compactnesss; JUnit XML would be more accurate. + + Args: + xunit_url_a: URL of the first XUnit file. + xunit_url_b: URL of the second XUnit file. + metadata: Optional metadata to include in the comparison result. + + Returns: + The comparison result as an XUnitComparison object. + + Raises: + HTTPError: If downloading either of the XUnit files fails. + ValueError: If the XUnit files contain different test suites and cannot be compared. + test suite identity is determined by (name, arch) pairs. + """ + if metadata is None: + metadata = {} + + async with aiohttp.ClientSession() as session: + + async def fetch_url(url: str) -> str: + async with session.get(url) as response: + response.raise_for_status() + return await response.text() + + xml_content_a, xml_content_b = await asyncio.gather( + fetch_url(xunit_url_a), + fetch_url(xunit_url_b), + ) + + test_suites_a = parse_xunit(xml_content_a) + test_suites_b = parse_xunit(xml_content_b) + + test_suite_a_keys = {(suite.name, suite.arch) for suite in test_suites_a} + test_suite_b_keys = {(suite.name, suite.arch) for suite in test_suites_b} + + if test_suite_a_keys != test_suite_b_keys: + raise ValueError( + "XUnit files contain different test suites and cannot be compared:\n" + + "\n".join( + ( + f"File A test suites: {test_suite_a_keys}", + f"File B test suites: {test_suite_b_keys}", + ) + ) + ) + + output: XUnitComparison = XUnitComparison( + status=XUnitComparisonStatus( + generated=True, # If we return this object, comparison was successful + reason="Comparison generated successfully", + ), + metadata=dict(metadata), + ) + + # Iterate over each test suite by matching (name, arch) pairs; if a test + # suite has multiple suites with the same (name, arch), the first one + # is arbitrarily used. + for name, arch in test_suite_a_keys: + suite_a = next(suite for suite in test_suites_a if suite.name == name and suite.arch == arch) + suite_b = next(suite for suite in test_suites_b if suite.name == name and suite.arch == arch) + + compare_test_suites(suite_a, suite_b, output) + + return output diff --git a/ymir/common/constants.py b/ymir/common/constants.py index bfdd8d7e..70b1c308 100644 --- a/ymir/common/constants.py +++ b/ymir/common/constants.py @@ -1,7 +1,16 @@ +from datetime import UTC, datetime, timedelta from enum import Enum JIRA_SEARCH_PATH = "rest/api/3/search/jql" +# Compares correctly - all our dates are tz-aware +DATETIME_MIN_UTC = datetime.min.replace(tzinfo=UTC) +# Groups within the redhat organization where we can find issues +GITLAB_GROUPS = ["rhel/rpms", "centos-stream/rpms"] +# Timeout for post-push testing (e.g., CAT tests) after stage push completes +POST_PUSH_TESTING_TIMEOUT = timedelta(hours=3) +POST_PUSH_TESTING_TIMEOUT_STR = "3 hours" + class RedisQueues(Enum): """Constants for Redis queue names used by Ymir agents""" diff --git a/ymir/common/models.py b/ymir/common/models.py index e657a8ce..0f779fe3 100644 --- a/ymir/common/models.py +++ b/ymir/common/models.py @@ -8,7 +8,7 @@ from datetime import datetime from enum import Enum, StrEnum from pathlib import Path -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, Field, RootModel @@ -670,3 +670,217 @@ class MergeRequestDetails(BaseModel): description: str = Field(description="Description of the MR") last_updated_at: datetime = Field(description="Timestamp of the last update (push)") comments: MergeRequestComments = Field(description="List of relevant MR comments") + + +# ============================================================================ +# Supervisor / Issue Verification Types +# ============================================================================ + + +class IssueStatus(StrEnum): + NEW = "New" + PLANNING = "Planning" # RHEL only + REFINEMENT = "Refinement" # RHELMISC only + IN_PROGRESS = "In Progress" + INTEGRATION = "Integration" # RHEL only + RELEASE_PENDING = "Release Pending" # RHEL only + DONE = "Done" # RHEL ONLY + CLOSED = "Closed" + + +class TestCoverage(StrEnum): + MANUAL = "Manual" + AUTOMATED = "Automated" + REGRESSION_ONLY = "RegressionOnly" + NEW_TEST_COVERAGE = "New Test Coverage" + + +class PreliminaryTesting(StrEnum): + REQUESTED = "Requested" + FAIL = "Fail" + PASS = "Pass" + READY = "Ready" + + +class ErrataStatus(StrEnum): + NEW_FILES = "NEW_FILES" + QE = "QE" + REL_PREP = "REL_PREP" + PUSH_READY = "PUSH_READY" + IN_PUSH = "IN_PUSH" + DROPPED_NO_SHIP = "DROPPED_NO_SHIP" + SHIPPED_LIVE = "SHIPPED_LIVE" + + +class ErrataComment(BaseModel): + authorName: str + authorEmail: str | None + created: datetime + body: str + + +class Erratum(BaseModel): + id: int + full_advisory: str + url: str + synopsis: str + status: ErrataStatus + jira_issues: list[str] + release_id: int + publish_date: datetime | None + last_status_transition_timestamp: datetime + assigned_to_email: str + package_owner_email: str + + +class FullErratum(Erratum): + comments: list[ErrataComment] | None = None + + +class GitlabMergeRequestState(StrEnum): + OPEN = "opened" + CLOSED = "closed" + MERGED = "merged" + + +class GitlabMergeRequest(BaseModel): + project: str + iid: int + url: str + title: str + description: str + state: GitlabMergeRequestState + merged_at: datetime | None + + +class Issue(BaseModel): + """A representation of a JIRA issue, with fields that we care about for RHEL development. + + RHEL development occurs in two JIRA projects - RHELMISC and RHEL, while many fields + are standard in JIRA or common to both, some fields will only be populated for RHEL issues. + + Defects and enhancements are covered in the RHEL project, the RHELMISC project is used for + tracking related activities of various types; we'll use issues in RHELMISC to tag Errata for + human attention. + """ + + key: str + url: str + assigned_team: str | None = None + summary: str + components: list[str] + status: IssueStatus + labels: list[str] + fix_versions: list[str] + errata_link: str | None # RHEL only + fixed_in_build: str | None = None # RHEL only + test_coverage: list[TestCoverage] | None = None # RHEL only + preliminary_testing: PreliminaryTesting | None = None # RHEL only + + +class JiraComment(ErrataComment): + id: str + + +class FullIssue(Issue): + description: str + comments: list[JiraComment] + + +class TestingFarmRequestState(StrEnum): + NEW = "new" + QUEUED = "queued" + RUNNING = "running" + ERROR = "error" + CANCELED = "canceled" + CANCEL_REQUESTED = "cancel-requested" + COMPLETE = "complete" + + +class TestingFarmRequestResult(StrEnum): + PASSED = "passed" + FAILED = "failed" + SKIPPED = "skipped" + UNKNOWN = "unknown" + ERROR = "error" + + +class TestingFarmRequest(BaseModel): + id: str + url: str + state: TestingFarmRequestState + result: TestingFarmRequestResult = TestingFarmRequestResult.UNKNOWN + error_reason: str | None = None + result_xunit_url: str | None = None + created: datetime + updated: datetime + + # We save the raw data to use during test reproduction + test_data: dict[str, Any] + environments_data: list[dict[str, Any]] + + @property + def arches(self) -> list[str]: + return [env["arch"] for env in self.environments_data] + + @property + def build_nvr(self) -> str: + versions = { + variables["BUILDS"] + for env in self.environments_data + if (variables := env.get("variables")) and "BUILDS" in variables + } + if len(versions) == 1: + return versions.pop() + + artifacts = { + id + for env in self.environments_data + for artifact in env.get("artifacts", []) + if artifact.get("type") == "redhat-brew-build" and (id := artifact.get("id")) + } + if len(artifacts) == 1: + return artifacts.pop() + + raise ValueError("Can't determine package version for request") + + +class YmirTag(BaseModel): + """A magic string appearing in the description of an issue that + associates it with a particular resource - like an erratum. + + This method of labelling issues and the format is borrowed from NEWA. + Using a custom field would be cleaner. + """ + + type: Literal["needs_attention"] + resource: Literal["erratum"] + id: str + + _LEGACY_PREFIXES = ("JOTNAR",) + + def __str__(self) -> str: + return f"::: YMIR {self.type} E: {self.id.strip()} :::" + + def all_formats(self) -> list[str]: + """Current and legacy tag strings for backwards-compatible search.""" + return [str(self)] + [f"::: {p} {self.type} E: {self.id.strip()} :::" for p in self._LEGACY_PREFIXES] + + +class TestingState(StrEnum): + NOT_RUNNING = "tests-not-running" + PENDING = "tests-pending" + RUNNING = "tests-running" + ERROR = "tests-error" + FAILED = "tests-failed" + PASSED = "tests-passed" + WAIVED = "tests-waived" + + +class WorkflowResult(BaseModel): + """Represents the result of running a workflow once.""" + + status: str = Field(description="A message describing what happened during the workflow run and why") + reschedule_in: float = Field( + description="Delay in seconds to reschedule the work item. Negative value means don't reschedule" + ) diff --git a/ymir/tools/privileged/errata.py b/ymir/tools/privileged/errata.py new file mode 100644 index 00000000..a1b3a172 --- /dev/null +++ b/ymir/tools/privileged/errata.py @@ -0,0 +1,185 @@ +import asyncio +import logging +import os +from datetime import UTC, datetime +from functools import cache +from typing import Any + +import requests +from beeai_framework.context import RunContext +from beeai_framework.emitter import Emitter +from beeai_framework.tools import JSONToolOutput, Tool, ToolError, ToolRunOptions +from pydantic import BaseModel, Field +from requests_gssapi import HTTPSPNEGOAuth + +from ymir.common.models import ErrataComment, ErrataStatus, Erratum, FullErratum + +logger = logging.getLogger(__name__) + +ET_URL = "https://errata.engineering.redhat.com" + + +@cache +def _et_verify() -> bool | str: + verify = os.getenv("REDHAT_IT_CA_BUNDLE") + if verify: + return verify + return True + + +def _et_api_get(path: str, *, params: dict | None = None) -> Any: + response = requests.get( + f"{ET_URL}/api/v1/{path}", + auth=HTTPSPNEGOAuth(opportunistic_auth=True), + verify=_et_verify(), + params=params, + ) + response.raise_for_status() + return response.json() + + +def _get_utc_timestamp_from_str(timestamp_string: str) -> datetime: + return datetime.strptime(timestamp_string, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC) + + +@cache +def _get_errata_user_email(id: int | str) -> str: + response = _et_api_get(f"user/{id}") + return response["login_name"] + + +def _get_erratum(erratum_id: str | int, *, full: bool = False) -> Erratum | FullErratum: + data = _et_api_get(f"erratum/{erratum_id}") + erratum_data = data["errata"] + + if "rhba" in erratum_data: + details = erratum_data["rhba"] + elif "rhsa" in erratum_data: + details = erratum_data["rhsa"] + elif "rhea" in erratum_data: + details = erratum_data["rhea"] + else: + raise ValueError("Unknown erratum type") + + jira_issues = [i["jira_issue"]["key"] for i in data["jira_issues"]["jira_issues"]] + + last_status_transition_timestamp = _get_utc_timestamp_from_str(details["status_updated_at"]) + publish_date = ( + _get_utc_timestamp_from_str(details["publish_date"]) if details["publish_date"] is not None else None + ) + assigned_to_email = _get_errata_user_email(details["assigned_to_id"]) + package_owner_email = _get_errata_user_email(details["package_owner_id"]) + + base = Erratum( + id=details["id"], + full_advisory=details["fulladvisory"], + url=f"{ET_URL}/advisory/{erratum_id}", + synopsis=details["synopsis"], + status=ErrataStatus(details["status"]), + jira_issues=jira_issues, + release_id=details["group_id"], + publish_date=publish_date, + last_status_transition_timestamp=last_status_transition_timestamp, + assigned_to_email=assigned_to_email, + package_owner_email=package_owner_email, + ) + + if full: + comments = _get_erratum_comments(erratum_id) + return FullErratum(**base.__dict__, comments=comments) + return base + + +def _get_erratum_comments(erratum_id: str | int) -> list[ErrataComment] | None: + data = _et_api_get(f"comments?filter[errata_id]={erratum_id}") + return [ + ErrataComment( + authorName=comment_data["attributes"]["who"]["realname"], + authorEmail=comment_data["attributes"]["who"]["login_name"], + created=datetime.fromisoformat(comment_data["attributes"]["created_at"].replace("Z", "+00:00")), + body=comment_data["attributes"]["text"], + ) + for comment_data in data["data"] + ] + + +# -- MCP Tools -- + + +class GetErratumToolInput(BaseModel): + erratum_id: str = Field(description="Erratum ID or advisory link (e.g. '12345' or full URL)") + full: bool = Field(default=False, description="If true, include comments in the response") + + +class GetErratumTool(Tool[GetErratumToolInput, ToolRunOptions, JSONToolOutput[dict[str, Any]]]): + name = "get_erratum" + description = """ + Get erratum details (basic or full with comments) by ID or link. + """ + input_schema = GetErratumToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "errata", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: GetErratumToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> JSONToolOutput[dict[str, Any]]: + erratum_id = tool_input.erratum_id + # Handle URL input — extract the ID from the end + if "/" in erratum_id: + erratum_id = erratum_id.rstrip("/").split("/")[-1] + + logger.info("Getting erratum %s (full=%s)", erratum_id, tool_input.full) + try: + erratum = await asyncio.to_thread(_get_erratum, erratum_id, full=tool_input.full) + except Exception as e: + raise ToolError(f"Failed to get erratum {erratum_id}: {e}") from e + + return JSONToolOutput(result=erratum.model_dump(mode="json")) + + +class GetErratumBuildNvrToolInput(BaseModel): + erratum_id: str = Field(description="Erratum ID") + package_name: str = Field(description="Package name to look up the build NVR for") + + +class GetErratumBuildNvrTool(Tool[GetErratumBuildNvrToolInput, ToolRunOptions, JSONToolOutput[str | None]]): + name = "get_erratum_build_nvr" + description = """ + Get the build NVR for a package in an erratum. + """ + input_schema = GetErratumBuildNvrToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "errata", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: GetErratumBuildNvrToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> JSONToolOutput[str | None]: + erratum_id = tool_input.erratum_id + package_name = tool_input.package_name + logger.info("Getting build NVR for %s in erratum %s", package_name, erratum_id) + + try: + builds_by_release = await asyncio.to_thread(_et_api_get, f"erratum/{erratum_id}/builds_list") + for release_info in builds_by_release.values(): + for builds_map in release_info["builds"]: + for build_nvr in builds_map: + if build_nvr.rsplit("-", 2)[0] == package_name: + return JSONToolOutput(result=build_nvr) + except Exception as e: + raise ToolError(f"Failed to get build NVR for {package_name} in erratum {erratum_id}: {e}") from e + + return JSONToolOutput(result=None) diff --git a/ymir/tools/privileged/gateway.py b/ymir/tools/privileged/gateway.py index da422962..f2bf024c 100644 --- a/ymir/tools/privileged/gateway.py +++ b/ymir/tools/privileged/gateway.py @@ -14,6 +14,7 @@ from ymir.tools.gateway_utils import setup_logging from ymir.tools.privileged.copr import BuildPackageTool, DownloadArtifactsTool from ymir.tools.privileged.distgit import CreateZstreamBranchTool +from ymir.tools.privileged.errata import GetErratumBuildNvrTool, GetErratumTool from ymir.tools.privileged.gitlab import ( AddBlockingMergeRequestCommentTool, AddMergeRequestCommentTool, @@ -29,18 +30,22 @@ OpenMergeRequestTool, PushToRemoteRepositoryTool, RetryPipelineJobTool, + SearchGitlabProjectMrsTool, ) from ymir.tools.privileged.jira import ( + AddJiraAttachmentsTool, AddJiraCommentTool, ChangeJiraStatusTool, CheckCveTriageEligibilityTool, EditJiraLabelsTool, + GetJiraAttachmentTool, GetJiraDetailsTool, GetJiraDevStatusTool, GetJiraPullRequestsTool, SearchJiraIssuesTool, SetJiraFieldsTool, SetPreliminaryTestingTool, + UpdateJiraCommentTool, VerifyIssueAuthorTool, ) from ymir.tools.privileged.logdetective import AnalyzeLogsTool @@ -50,6 +55,10 @@ UploadSourcesTool, ) from ymir.tools.privileged.maintainer_rules import MaintainerRulesTool +from ymir.tools.privileged.testing_farm import ( + GetTestingFarmRequestTool, + ReproduceTestingFarmRequestTool, +) from ymir.tools.privileged.zstream_search import ZStreamSearchTool # Patterns that match common credential formats in log output @@ -128,16 +137,24 @@ def main(): PushToRemoteRepositoryTool(), RetryPipelineJobTool(), FetchGitlabMrNotesTool(), + SearchGitlabProjectMrsTool(), + GetErratumTool(), + GetErratumBuildNvrTool(), + GetTestingFarmRequestTool(), + ReproduceTestingFarmRequestTool(), + AddJiraAttachmentsTool(), AddJiraCommentTool(), ChangeJiraStatusTool(), CheckCveTriageEligibilityTool(), EditJiraLabelsTool(), + GetJiraAttachmentTool(), GetJiraDetailsTool(), GetJiraDevStatusTool(), GetJiraPullRequestsTool(), SearchJiraIssuesTool(), SetJiraFieldsTool(), SetPreliminaryTestingTool(), + UpdateJiraCommentTool(), VerifyIssueAuthorTool(), DownloadSourcesTool(), PrepSourcesTool(), diff --git a/ymir/tools/privileged/gitlab.py b/ymir/tools/privileged/gitlab.py index 1f57fc2e..5ab87181 100644 --- a/ymir/tools/privileged/gitlab.py +++ b/ymir/tools/privileged/gitlab.py @@ -3,6 +3,7 @@ import logging import os import re +from typing import Any from urllib.parse import quote, urlparse import aiohttp @@ -1023,3 +1024,83 @@ async def _run( except Exception as e: logger.error("Error fetching GitLab MR notes: %s", e) return StringToolOutput(result=f"Error fetching GitLab MR notes: {e}") + + +class SearchGitlabProjectMrsToolInput(BaseModel): + project: str = Field(description="GitLab project path (e.g. 'redhat/rhel/rpms/podman')") + search: str = Field(description="Search string to match against merge requests (e.g. a JIRA issue key)") + state: str | None = Field( + default=None, + description="Filter MRs by state: 'opened', 'closed', 'merged', or null for all", + ) + + +class SearchGitlabProjectMrsTool( + Tool[ + SearchGitlabProjectMrsToolInput, + ToolRunOptions, + JSONToolOutput[list[dict[str, Any]]], + ] +): + name = "search_gitlab_project_mrs" + description = """ + Searches for merge requests in a GitLab project matching a search string + (typically a JIRA issue key). Returns a list of matching MRs with their + project, iid, url, title, description, state, and merged_at timestamp. + """ + input_schema = SearchGitlabProjectMrsToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "gitlab", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: SearchGitlabProjectMrsToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> JSONToolOutput[list[dict[str, Any]]]: + project = tool_input.project + search = tool_input.search + state = tool_input.state + + encoded_project = quote(project, safe="") + url = f"https://gitlab.com/api/v4/projects/{encoded_project}/merge_requests" + + params: dict[str, str] = {"search": search} + if state is not None: + params["state"] = state + + headers = _get_auth_headers(f"https://gitlab.com/{project}") + logger.info("Searching MRs for %s in %s (state=%s)", search, project, state) + + try: + async with ( + aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session, + session.get(url, headers=headers, params=params) as response, + ): + response.raise_for_status() + data = await response.json() + + results = [ + { + "project": project, + "iid": mr["iid"], + "url": mr["web_url"], + "title": mr["title"], + "description": mr.get("description", ""), + "state": mr["state"], + "merged_at": mr.get("merged_at"), + } + for mr in data + ] + + logger.info("Found %d MR(s) for %s in %s", len(results), search, project) + return JSONToolOutput(result=results) + + except Exception as e: + from beeai_framework.tools import ToolError + + raise ToolError(f"Failed to search MRs in {project}: {e}") from e diff --git a/ymir/tools/privileged/jira.py b/ymir/tools/privileged/jira.py index b23781e6..4dab8d9f 100644 --- a/ymir/tools/privileged/jira.py +++ b/ymir/tools/privileged/jira.py @@ -981,13 +981,13 @@ class GetJiraDevStatusToolInput(BaseModel): class GetJiraDevStatusTool( - Tool[GetJiraDevStatusToolInput, ToolRunOptions, JSONToolOutput[list[dict[str, Any]]]] + Tool[GetJiraDevStatusToolInput, ToolRunOptions, JSONToolOutput[dict[str, list[dict[str, Any]]]]] ): name = "get_jira_dev_status" description = """ Gets development status (linked commits) for a Jira issue using the - Jira Dev-Status API. Returns a list of commit objects with - url, message, and repository_url fields. + Jira Dev-Status API. Returns a dictionary with a 'commits' key containing + a list of commit objects with url, message, and repository_url fields. """ input_schema = GetJiraDevStatusToolInput @@ -1002,7 +1002,7 @@ async def _run( tool_input: GetJiraDevStatusToolInput, options: ToolRunOptions | None, context: RunContext, - ) -> JSONToolOutput[list[dict[str, Any]]]: + ) -> JSONToolOutput[dict[str, list[dict[str, Any]]]]: issue_key = tool_input.issue_key headers = get_jira_auth_headers() jira_base = os.getenv("JIRA_URL") @@ -1030,7 +1030,7 @@ async def _run( ] logger.info(f"Found {len(commits)} commits in development status for {issue_key}") - return JSONToolOutput(result=commits) + return JSONToolOutput(result={"commits": commits}) class GetJiraPullRequestsToolInput(BaseModel): @@ -1041,14 +1041,15 @@ class GetJiraPullRequestsTool( Tool[ GetJiraPullRequestsToolInput, ToolRunOptions, - JSONToolOutput[list[dict[str, Any]]], + JSONToolOutput[dict[str, list[dict[str, Any]]]], ] ): name = "get_jira_pull_requests" description = """ Gets pull/merge requests linked to a Jira issue via the dev-status API. - Returns a list of pull request dicts with keys: id, name, status, url, - source, destination, repositoryName, repositoryUrl. + Returns a dictionary with a 'pull_requests' key containing a list of pull + request dicts with keys: id, name, status, url, source, destination, + repositoryName, repositoryUrl. """ input_schema = GetJiraPullRequestsToolInput @@ -1063,7 +1064,7 @@ async def _run( tool_input: GetJiraPullRequestsToolInput, options: ToolRunOptions | None, context: RunContext, - ) -> JSONToolOutput[list[dict[str, Any]]]: + ) -> JSONToolOutput[dict[str, list[dict[str, Any]]]]: issue_key = tool_input.issue_key headers = get_jira_auth_headers() jira_base = os.getenv("JIRA_URL") @@ -1084,7 +1085,7 @@ async def _run( pull_requests.extend(detail.get("pullRequests", [])) logger.info(f"Found {len(pull_requests)} pull requests for {issue_key}") - return JSONToolOutput(result=pull_requests) + return JSONToolOutput(result={"pull_requests": pull_requests}) class SetPreliminaryTestingToolInput(BaseModel): @@ -1158,3 +1159,202 @@ async def _run( return StringToolOutput( result=f"Successfully set Preliminary Testing to {value.value} on {issue_key}" ) + + +class UpdateJiraCommentToolInput(BaseModel): + issue_key: str = Field(description="Jira issue key (e.g. RHEL-12345)") + comment_id: str = Field(description="ID of the comment to update") + comment: str = Field(description="New comment text") + + +class UpdateJiraCommentTool(Tool[UpdateJiraCommentToolInput, ToolRunOptions, StringToolOutput]): + name = "update_jira_comment" + description = """ + Updates an existing comment on a Jira issue. + """ + input_schema = UpdateJiraCommentToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "jira", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: UpdateJiraCommentToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> StringToolOutput: + issue_key = tool_input.issue_key + comment_id = tool_input.comment_id + comment = tool_input.comment + + if os.getenv("DRY_RUN", "False").lower() == "true": + return StringToolOutput( + result=f"Dry run, not updating comment {comment_id} on {issue_key} " + f"(this is expected, not an error)" + ) + if _skip_jira_writes(): + return StringToolOutput( + result=f"JIRA_DRY_RUN is set, not updating comment {comment_id} " + f"on {issue_key} (this is expected, not an error)" + ) + + jira_url = urljoin(os.getenv("JIRA_URL"), f"rest/api/2/issue/{issue_key}/comment/{comment_id}") + logger.info("Updating comment %s on %s", comment_id, issue_key) + + async with aiohttpClientSession(timeout=AIOHTTP_TIMEOUT) as session: + try: + async with session.put( + jira_url, + json={"body": comment}, + headers=get_jira_auth_headers(), + ) as response: + response.raise_for_status() + except aiohttp.ClientError as e: + raise ToolError(f"Failed to update comment {comment_id} on {issue_key}: {e}") from e + + return StringToolOutput(result=f"Successfully updated comment {comment_id} on {issue_key}") + + +class AddJiraAttachmentsToolInput(BaseModel): + issue_key: str = Field(description="Jira issue key (e.g. RHEL-12345)") + attachments: list[dict[str, str]] = Field( + description="List of attachments, each with 'filename' and 'content' (text content) keys" + ) + + +class AddJiraAttachmentsTool(Tool[AddJiraAttachmentsToolInput, ToolRunOptions, StringToolOutput]): + name = "add_jira_attachments" + description = """ + Adds attachments to a Jira issue. + """ + input_schema = AddJiraAttachmentsToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "jira", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: AddJiraAttachmentsToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> StringToolOutput: + issue_key = tool_input.issue_key + attachments = tool_input.attachments + + if not attachments: + return StringToolOutput(result=f"No attachments to add to {issue_key}") + + if os.getenv("DRY_RUN", "False").lower() == "true": + filenames = ", ".join(a["filename"] for a in attachments) + return StringToolOutput( + result=f"Dry run, not adding attachments ({filenames}) to {issue_key} " + f"(this is expected, not an error)" + ) + if _skip_jira_writes(): + return StringToolOutput( + result=f"JIRA_DRY_RUN is set, not adding attachments " + f"to {issue_key} (this is expected, not an error)" + ) + + jira_url = urljoin(os.getenv("JIRA_URL"), f"rest/api/2/issue/{issue_key}/attachments") + headers = dict(get_jira_auth_headers()) + # Remove Content-Type so aiohttp sets multipart correctly + headers.pop("Content-Type", None) + headers["X-Atlassian-Token"] = "no-check" + + logger.info("Adding %d attachment(s) to %s", len(attachments), issue_key) + + async with aiohttpClientSession(timeout=AIOHTTP_TIMEOUT) as session: + try: + data = aiohttp.FormData() + for attachment in attachments: + content = attachment["content"].encode("utf-8") + data.add_field( + "file", + content, + filename=attachment["filename"], + content_type="text/plain", + ) + + async with session.post( + jira_url, + data=data, + headers=headers, + ) as response: + response.raise_for_status() + except aiohttp.ClientError as e: + raise ToolError(f"Failed to add attachments to {issue_key}: {e}") from e + + filenames = ", ".join(a["filename"] for a in attachments) + return StringToolOutput(result=f"Successfully added attachments ({filenames}) to {issue_key}") + + +class GetJiraAttachmentToolInput(BaseModel): + issue_key: str = Field(description="Jira issue key (e.g. RHEL-12345)") + filename: str = Field(description="Filename of the attachment to download") + + +class GetJiraAttachmentTool(Tool[GetJiraAttachmentToolInput, ToolRunOptions, StringToolOutput]): + name = "get_jira_attachment" + description = """ + Downloads an attachment from a Jira issue by filename and returns its text content. + """ + input_schema = GetJiraAttachmentToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "jira", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: GetJiraAttachmentToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> StringToolOutput: + issue_key = tool_input.issue_key + filename = tool_input.filename + headers = get_jira_auth_headers() + jira_base = os.getenv("JIRA_URL") + + logger.info("Downloading attachment %s from %s", filename, issue_key) + + async with aiohttpClientSession(timeout=AIOHTTP_TIMEOUT) as session: + # Get issue attachments + issue_url = urljoin(jira_base, f"rest/api/2/issue/{issue_key}?fields=attachment") + try: + async with session.get(issue_url, headers=headers) as response: + response.raise_for_status() + issue_data = await response.json() + except aiohttp.ClientError as e: + raise ToolError(f"Failed to get issue {issue_key}: {e}") from e + + attachments = issue_data.get("fields", {}).get("attachment", []) + matching = [a for a in attachments if a.get("filename") == filename] + + if len(matching) == 0: + raise ToolError(f"Issue {issue_key} has no attachment named {filename}") + if len(matching) > 1: + raise ToolError(f"Issue {issue_key} has multiple attachments named {filename}") + + content_url = matching[0]["content"] + try: + async with session.get(content_url, headers=headers) as response: + response.raise_for_status() + content = await response.read() + except aiohttp.ClientError as e: + raise ToolError(f"Failed to download attachment {filename}: {e}") from e + + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + return StringToolOutput(result=f"Failed to decode attachment {filename} as UTF-8") + + return StringToolOutput(result=text) diff --git a/ymir/tools/privileged/testing_farm.py b/ymir/tools/privileged/testing_farm.py new file mode 100644 index 00000000..b82144ce --- /dev/null +++ b/ymir/tools/privileged/testing_farm.py @@ -0,0 +1,195 @@ +import logging +import os +from datetime import datetime +from functools import cache +from json import dumps as json_dumps +from typing import Any + +import requests +from beeai_framework.context import RunContext +from beeai_framework.emitter import Emitter +from beeai_framework.tools import JSONToolOutput, Tool, ToolError, ToolRunOptions +from pydantic import BaseModel, Field + +from ymir.common.models import ( + TestingFarmRequest, + TestingFarmRequestResult, +) + +logger = logging.getLogger(__name__) + +TESTING_FARM_URL = "https://api.testing-farm.io/v0.1" + + +@cache +def _testing_farm_headers() -> dict[str, str]: + token = os.environ["TESTING_FARM_API_TOKEN"] + return { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + } + + +def _testing_farm_api_get(path: str, *, params: dict | None = None) -> Any: + url = f"{TESTING_FARM_URL}/{path}" + response = requests.get(url, headers=_testing_farm_headers(), params=params) + if not response.ok: + logger.error( + "GET %s%s failed.\nerror:\n%s", url, f" (params={params})" if params else "", response.text + ) + response.raise_for_status() + return response.json() + + +def _testing_farm_api_post(path: str, json: dict[str, Any]) -> Any: + url = f"{TESTING_FARM_URL}/{path}" + response = requests.post(url, headers=_testing_farm_headers(), json=json) + if not response.ok: + logger.error( + "POST to %s failed\nbody:\n%s\nerror:\n%s", url, json_dumps(json, indent=2), response.text + ) + response.raise_for_status() + return response.json() + + +def _parse_tf_request(response: dict[str, Any]) -> TestingFarmRequest: + result_data = response.get("result") + result = result_data["overall"] if result_data else TestingFarmRequestResult.UNKNOWN + error_reason = result_data.get("summary") if result == TestingFarmRequestResult.ERROR else None + + return TestingFarmRequest( + id=response["id"], + url=f"{TESTING_FARM_URL}/requests/{response['id']}", + state=response["state"], + result=result, + error_reason=error_reason, + result_xunit_url=result_data.get("xunit_url") if result_data else None, + created=datetime.fromisoformat(response["created"]), + updated=datetime.fromisoformat(response["updated"]), + test_data=response.get("test", {}), + environments_data=response.get("environments_requested", response.get("environments", [])), + ) + + +# -- MCP Tools -- + + +class GetTestingFarmRequestToolInput(BaseModel): + request_id: str = Field(description="Testing Farm request ID") + + +class GetTestingFarmRequestTool( + Tool[GetTestingFarmRequestToolInput, ToolRunOptions, JSONToolOutput[dict[str, Any]]] +): + name = "get_testing_farm_request" + description = """ + Get a Testing Farm request by ID. + """ + input_schema = GetTestingFarmRequestToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "testing_farm", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: GetTestingFarmRequestToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> JSONToolOutput[dict[str, Any]]: + logger.info("Getting Testing Farm request %s", tool_input.request_id) + try: + response = _testing_farm_api_get(f"requests/{tool_input.request_id}") + tf_request = _parse_tf_request(response) + except Exception as e: + raise ToolError(f"Failed to get Testing Farm request {tool_input.request_id}: {e}") from e + + return JSONToolOutput(result=tf_request.model_dump(mode="json")) + + +class ReproduceTestingFarmRequestToolInput(BaseModel): + request_id: str = Field(description="ID of the original Testing Farm request to reproduce") + build_nvr: str = Field(description="NVR of the build to use for reproduction") + + +class ReproduceTestingFarmRequestTool( + Tool[ReproduceTestingFarmRequestToolInput, ToolRunOptions, JSONToolOutput[dict[str, Any]]] +): + name = "reproduce_testing_farm_request" + description = """ + Reproduce a Testing Farm request with a different build NVR. + """ + input_schema = ReproduceTestingFarmRequestToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "testing_farm", self.name], + creator=self, + ) + + async def _run( + self, + tool_input: ReproduceTestingFarmRequestToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> JSONToolOutput[dict[str, Any]]: + request_id = tool_input.request_id + build_nvr = tool_input.build_nvr + logger.info("Reproducing Testing Farm request %s with build %s", request_id, build_nvr) + + if os.getenv("DRY_RUN", "False").lower() == "true": + return JSONToolOutput( + result={ + "id": f"dry-run-{request_id}", + "message": f"Dry run: would reproduce {request_id} with build {build_nvr}", + } + ) + + try: + # Fetch the original request + original_response = _testing_farm_api_get(f"requests/{request_id}") + original = _parse_tf_request(original_response) + + # Build new environments with the replacement build + def create_new_environment(env: dict) -> dict: + new_env = { + "arch": env["arch"], + "os": env["os"], + "tmt": { + "context": { + k: v for k, v in env["tmt"]["context"].items() if not k.startswith("newa_") + } + }, + } + + builds_var = env.get("variables", {}).get("BUILDS") + if builds_var is not None: + new_env["variables"] = env["variables"] | {"BUILDS": build_nvr} + return new_env + + new_env["variables"] = env.get("variables", {}) + + artifacts = env.get("artifacts") + if artifacts and len(artifacts) == 1: + new_env["artifacts"] = [{"id": build_nvr, "type": "redhat-brew-build", "order": 40}] + return new_env + + raise ToolError( + "Cannot reproduce Testing Farm request: " + "cannot determine how to replace build in environment." + ) + + body = { + "test": original.test_data, + "environments": [create_new_environment(env) for env in original.environments_data], + } + + response = _testing_farm_api_post("requests", json=body) + new_request = _parse_tf_request(response) + + except Exception as e: + raise ToolError(f"Failed to reproduce Testing Farm request {request_id}: {e}") from e + + return JSONToolOutput(result=new_request.model_dump(mode="json")) diff --git a/ymir/tools/privileged/zstream_search.py b/ymir/tools/privileged/zstream_search.py index b5d21dfc..5401c657 100644 --- a/ymir/tools/privileged/zstream_search.py +++ b/ymir/tools/privileged/zstream_search.py @@ -236,7 +236,8 @@ def _not_found(): GetJiraDevStatusTool(), issue_key=issue_key, ) - commits = json.loads(dev_status) if isinstance(dev_status, str) else dev_status + dev_status_data = json.loads(dev_status) if isinstance(dev_status, str) else dev_status + commits = dev_status_data.get("commits", []) except Exception as e: logger.debug(f"Dev status request error for {issue_key}: {e}") continue diff --git a/ymir/tools/unprivileged/analyze_ewa_testrun.py b/ymir/tools/unprivileged/analyze_ewa_testrun.py new file mode 100644 index 00000000..56f77e75 --- /dev/null +++ b/ymir/tools/unprivileged/analyze_ewa_testrun.py @@ -0,0 +1,93 @@ +import asyncio +import logging +import re + +import nitrate +from beeai_framework.context import RunContext +from beeai_framework.emitter import Emitter +from beeai_framework.tools import StringToolOutput, Tool, ToolRunOptions +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +# Include only meaningful lines from the notes +NOTES_INCLUDE_PATTERN = re.compile(r"CR#|=>|-files|-avc|beaker-task") +# Unless called with --full, skip the Errata Workflow caseruns +CASERUN_EXCLUDE_PATTERN = re.compile(r"Errata Workflow") + + +def get_tcms_run_details(run_id: str, *, full: bool = False, color: bool = False) -> str: + """ + Fetches and filters TCMS test run details. + """ + if not color: + nitrate.set_color_mode(nitrate.COLOR_OFF) + else: + nitrate.set_color_mode(nitrate.COLOR_ON) + + testrun = nitrate.TestRun(int(run_id)) + output = [] + + for caserun in testrun.caseruns: + caserun_str = str(caserun) + notes_str = str(caserun.notes) + + passed = caserun.status == nitrate.Status("PASSED") + + output_entry = [] + if full: + output_entry.append(caserun_str) + output_entry.extend(notes_str.splitlines()) + else: + # filter out Errata Workflow and not useful lines from notes + if CASERUN_EXCLUDE_PATTERN.search(caserun_str): + continue + output_entry.append(caserun_str) + # add the details from notes only for tests that do not pass + if not passed: + output_entry.extend( + line for line in notes_str.splitlines() if NOTES_INCLUDE_PATTERN.search(line) + ) + if output_entry: + output.append(output_entry) + + # Flatten the list of lists into a single list of strings + flattened_output = [item for sublist in output for item in sublist] + # Join the strings into a single multiline string + return "\n".join(flattened_output) + + +class AnalyzeEwaTestRunToolInput(BaseModel): + run_id: int = Field(description="TCMS Test Run ID (e.g., 12345)") + + +class AnalyzeEwaTestRunTool(Tool[AnalyzeEwaTestRunToolInput, ToolRunOptions, StringToolOutput]): + name = "analyze_ewa_testrun" + description = ( + "Analyzes a TCMS test run generated by Errata Workflow Automation (EWA) " + "and creates a test-case by test-case report." + ) + input_schema = AnalyzeEwaTestRunToolInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "analyze_ewa_testrun"], + creator=self, + ) + + async def _run( + self, + input: AnalyzeEwaTestRunToolInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> StringToolOutput: + try: + # Fetch the TCMS run details + run_details = await asyncio.to_thread(get_tcms_run_details, input.run_id) + + # Return the formatted details + return StringToolOutput(result=run_details) + + except Exception as e: + logger.error(f"Failed to get TCMS run details for {input.run_id}: {e}") + return StringToolOutput(result=f"Error: Failed to get TCMS run details for {input.run_id}: {e!s}") diff --git a/ymir/tools/unprivileged/read_logfile.py b/ymir/tools/unprivileged/read_logfile.py new file mode 100644 index 00000000..0448ccdd --- /dev/null +++ b/ymir/tools/unprivileged/read_logfile.py @@ -0,0 +1,40 @@ +import logging + +import aiohttp +from beeai_framework.context import RunContext +from beeai_framework.emitter import Emitter +from beeai_framework.tools import StringToolOutput, Tool, ToolRunOptions +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + + +class ReadLogfileInput(BaseModel): + logfile_url: str = Field(description="URL of logfile to read") + + +class ReadLogfileTool(Tool[ReadLogfileInput, ToolRunOptions, StringToolOutput]): + name = "read_logfile" + description = "Read logfile from URL" + input_schema = ReadLogfileInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "read_logfile"], + creator=self, + ) + + async def _run( + self, + input: ReadLogfileInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> StringToolOutput: + logger.info("Reading logfile from URL: %s", input.logfile_url) + async with aiohttp.ClientSession() as session, session.get(input.logfile_url) as response: + if response.status == 200: + return StringToolOutput( + result=await response.text(), + ) + + return StringToolOutput(result=f"Failed to read logfile from {input.logfile_url}") diff --git a/ymir/tools/unprivileged/read_readme.py b/ymir/tools/unprivileged/read_readme.py new file mode 100644 index 00000000..5f2186c3 --- /dev/null +++ b/ymir/tools/unprivileged/read_readme.py @@ -0,0 +1,52 @@ +import logging + +import aiohttp +from beeai_framework.context import RunContext +from beeai_framework.emitter import Emitter +from beeai_framework.tools import StringToolOutput, Tool, ToolRunOptions +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + +README_PATTERNS = [ + ("https://gitlab.com/", "/-/raw/main/README.md?ref_type=heads&inline=false"), + ( + "https://gitlab.cee.redhat.com/", + "/-/raw/main/README.md?ref_type=heads&inline=false", + ), + ("https://pkgs.devel.redhat.com/", "/plain/README"), +] + + +class ReadReadmeInput(BaseModel): + repo_url: str = Field(description="URL of git repository to read README from") + + +class ReadReadmeTool(Tool[ReadReadmeInput, ToolRunOptions, StringToolOutput]): + name = "read_readme" + description = "Read README file from git repository" + input_schema = ReadReadmeInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "read_readme"], + creator=self, + ) + + async def _run( + self, + input: ReadReadmeInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> StringToolOutput: + async with aiohttp.ClientSession() as session: + for prefix, suffix in README_PATTERNS: + if input.repo_url.startswith(prefix): + url = input.repo_url.removesuffix("/") + suffix + async with session.get(url) as response: + if response.status == 200: + return StringToolOutput( + result=await response.text(), + ) + + return StringToolOutput(result=f"Failed to find README.md for {input.repo_url}") diff --git a/ymir/tools/unprivileged/search_resultsdb.py b/ymir/tools/unprivileged/search_resultsdb.py new file mode 100644 index 00000000..0d2ea2ac --- /dev/null +++ b/ymir/tools/unprivileged/search_resultsdb.py @@ -0,0 +1,148 @@ +import logging +from datetime import datetime +from enum import StrEnum +from urllib.parse import quote as urlquote + +import aiohttp +from beeai_framework.context import RunContext +from beeai_framework.emitter import Emitter +from beeai_framework.tools import Tool, ToolError, ToolOutput, ToolRunOptions +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + + +RESULTS_DB_URL = "https://resultsdb-api.engineering.redhat.com" + + +class ResultsdbOutput(StrEnum): + """The possible resultsdb outcome values, we only care about a subset of them.""" + + PASSED = "PASSED" + INFO = "INFO" + FAILED = "FAILED" + NEEDS_INSPECTION = "NEEDS_INSPECTION" + NOT_APPLICABLE = "NOT_APPLICABLE" + QUEUED = "QUEUED" + RUNNING = "RUNNING" + ERROR = "ERROR" + PENDING = "PENDING" + + +class ResultsDbResult(BaseModel): + """A small subset of the full resultsdb result schema, but it is all we need.""" + + testcase_name: str + outcome: ResultsdbOutput + ref_url: str | None + # Renamed from submit_time, since that sounds like when the test was + # started, but it means when the result was submitted to resultsdb + last_updated: datetime + + +# Arbitrary limit, we shouldn't get this many results for a single NVR +MAX_RESULTS = 100 + + +async def search_resultsdb(package_nvr: str, name_pattern: str) -> list[ResultsDbResult]: + url = ( + f"{RESULTS_DB_URL}/api/v2.0/results" + f"?item={urlquote(package_nvr)}" + f"&testcases:like={urlquote(name_pattern)}" + f"&limit={MAX_RESULTS}" + ) + logger.info("Fetching resultsdb data from %s", url) + async with aiohttp.ClientSession() as session, session.get(url) as response: + if response.status == 200: + response_json = await response.json() + + if response_json.get("next") is not None: + raise ValueError(f"ResultsDB returned more than {MAX_RESULTS} results") + + results: list[ResultsDbResult] = [ + ResultsDbResult( + testcase_name=r["testcase"]["name"], + outcome=ResultsdbOutput(r["outcome"]), + ref_url=r["ref_url"], + last_updated=datetime.fromisoformat(r["submit_time"]), + ) + for r in response_json.get("data", []) + ] + + # Now only keep the latest result for each testcase + latest_results: dict[str, ResultsDbResult] = {} + for r in results: + if ( + r.testcase_name not in latest_results + or r.last_updated > latest_results[r.testcase_name].last_updated + ): + latest_results[r.testcase_name] = r + + logger.info( + "Found %d results in resultsdb (%d after filtering for latest submissions)", + len(results), + len(latest_results), + ) + + return list(latest_results.values()) + text = await response.text() + raise aiohttp.ClientResponseError( + response.request_info, + response.history, + status=response.status, + message=text, + headers=response.headers, + ) + + +class SearchResultsdbInput(BaseModel): + package_nvr: str = Field(description="NVR of the package to search in the results database") + name_pattern: str = Field( + description="Pattern to search for in the testcase names, e.g. 'frontend.regression.test-%'" + ) + + +class SearchResultsdbOutput(BaseModel, ToolOutput): + results: list[ResultsDbResult] + + def get_text_content(self) -> str: + return self.model_dump_json(indent=2, exclude_unset=True) + + def is_empty(self) -> bool: + return len(self.results) == 0 + + +class SearchResultsdbTool(Tool[SearchResultsdbInput, ToolRunOptions, SearchResultsdbOutput]): + """ + Tool to search for results for a specific package build in resultsdb. + https://github.com/release-engineering/resultsdb + """ + + name = "search_resultsdb" + description = ( + "Search for results for a specific package build in resultsdb. " + "Only use this tool if the maintainer rules indicate that " + "test results will be in resultsdb." + ) + input_schema = SearchResultsdbInput + + def _create_emitter(self) -> Emitter: + return Emitter.root().child( + namespace=["tool", "search_resultsdb"], + creator=self, + ) + + async def _run( + self, + input: SearchResultsdbInput, + options: ToolRunOptions | None, + context: RunContext, + ) -> SearchResultsdbOutput: + try: + return SearchResultsdbOutput( + results=await search_resultsdb(input.package_nvr, input.name_pattern) + ) + except aiohttp.ClientError as e: + raise ToolError(f"Failed to fetch resultsdb data: {e}") from e + except ValueError as e: + raise ToolError(f"Error parsing resultsdb data: {e}") from e