diff --git a/.github/ci-doctor-mq/schemas/pattern.schema.json b/.github/ci-doctor-mq/schemas/pattern.schema.json index affd44ec3926..00caafb9b9b3 100644 --- a/.github/ci-doctor-mq/schemas/pattern.schema.json +++ b/.github/ci-doctor-mq/schemas/pattern.schema.json @@ -16,7 +16,8 @@ "last_seen", "recent_run_urls", "affected_prs", - "recent_timestamps" + "recent_timestamps", + "rerun_search_string" ], "properties": { "schema_version": { @@ -94,6 +95,14 @@ "type": "string", "format": "date-time" } + }, + "rerun_search_string": { + "type": [ + "string", + "null" + ], + "minLength": 1, + "description": "Stable, job-agnostic substring of the raw error (with absolute paths, line/column numbers, hex addresses, PIDs, timestamps, run IDs, commit SHAs, tmp dirs, and UUIDs stripped) that uniquely identifies this failure in run logs. Consumed by the workflow_rerunner (.github/scripts/workflow_rerun) to auto-rerun transient failures, similar to entries in errors_to_look_for.json. Populated only for transient, rerunnable categories (Flaky Test, Infrastructure, Network, External Service); null for all other categories where a plain rerun cannot fix the failure." } } } \ No newline at end of file diff --git a/.github/scripts/workflow_rerun/argument_parser.py b/.github/scripts/workflow_rerun/argument_parser.py index f62d9dffc4bb..ccac9cbadaf8 100644 --- a/.github/scripts/workflow_rerun/argument_parser.py +++ b/.github/scripts/workflow_rerun/argument_parser.py @@ -24,6 +24,13 @@ def get_arguments() -> argparse.Namespace: required=False, help='.json file with the errors to look for in logs', default=Path(__file__).resolve().parent.joinpath('errors_to_look_for.json')) + parser.add_argument('--patterns-dir', + type=Path, + required=False, + help='Directory with CI Doctor MQ pattern .json files (fetched from the ' + 'memory/ci-doctor-mq branch). Their "rerun_search_string" values are ' + 'used as additional errors to look for in logs', + default=None) parser.add_argument('--dry-run', required=False, action='store_true', diff --git a/.github/scripts/workflow_rerun/log_analyzer.py b/.github/scripts/workflow_rerun/log_analyzer.py index 49aef69fda1e..2a06c5e8b52f 100644 --- a/.github/scripts/workflow_rerun/log_analyzer.py +++ b/.github/scripts/workflow_rerun/log_analyzer.py @@ -5,11 +5,15 @@ import re import tempfile from pathlib import Path -from typing import TypedDict +from typing import Optional, TypedDict from zipfile import ZipFile from workflow_rerun.constants import LOGGER +# Sentinel ticket number used for errors sourced from CI Doctor MQ pattern files, +# which are not tied to a specific JIRA/GitHub ticket. +CI_DOCTOR_PATTERN_TICKET = 9999999 + class LogFile(TypedDict): file_name: str @@ -24,11 +28,14 @@ class ErrorData(TypedDict): class LogAnalyzer: def __init__(self, path_to_logs: Path, - path_to_errors_file: Path) -> None: + path_to_errors_file: Path, + patterns_dir: Optional[Path] = None) -> None: self._path_to_errors_file = path_to_errors_file + self._patterns_dir = patterns_dir self._errors_to_look_for: list[ErrorData] = [] self._collect_errors_to_look_for() + self._collect_errors_from_patterns() self._log_dir = path_to_logs @@ -52,6 +59,45 @@ def _collect_errors_to_look_for(self) -> None: ticket=error_data['ticket']) ) + def _collect_errors_from_patterns(self) -> None: + """ + Collects additional errors to look for from CI Doctor MQ pattern files. + + Each pattern .json file (fetched from the memory/ci-doctor-mq branch) may + contain a "rerun_search_string" field: a stable, searchable substring of a + transient failure's log output. When present and non-empty, it is used as an + additional error to look for, analogous to entries in errors_to_look_for.json. + Pattern-derived errors are not tied to a specific ticket, so they are recorded + with the CI_DOCTOR_PATTERN_TICKET sentinel ticket number. + """ + if self._patterns_dir is None: + return + + if not self._patterns_dir.is_dir(): + LOGGER.info(f'PATTERNS DIR {self._patterns_dir} DOES NOT EXIST, SKIPPING PATTERNS') + return + + for pattern_file in sorted(self._patterns_dir.glob('*.json')): + try: + with open(file=pattern_file, + mode='r', + encoding='utf-8') as _pattern_file: + pattern_data = json.load(_pattern_file) + except (json.JSONDecodeError, OSError) as error: + LOGGER.warning(f'COULD NOT READ PATTERN FILE {pattern_file}: {error}') + continue + + rerun_search_string = pattern_data.get('rerun_search_string') + if not isinstance(rerun_search_string, str) or not rerun_search_string.strip(): + continue + + LOGGER.info(f'ADDING RERUN SEARCH STRING FROM PATTERN {pattern_file.name}: ' + f'"{rerun_search_string}"') + self._errors_to_look_for.append( + ErrorData(error_text=rerun_search_string, + ticket=CI_DOCTOR_PATTERN_TICKET) + ) + def _collect_log_files(self) -> None: """ Collects the .txt log files from the log archive diff --git a/.github/scripts/workflow_rerun/rerunner.py b/.github/scripts/workflow_rerun/rerunner.py index e54cdfeae731..3f914ad62e1b 100644 --- a/.github/scripts/workflow_rerun/rerunner.py +++ b/.github/scripts/workflow_rerun/rerunner.py @@ -16,6 +16,7 @@ from workflow_rerun.argument_parser import get_arguments from workflow_rerun.constants import GITHUB_TOKEN, LOGGER from workflow_rerun.log_analyzer import LogAnalyzer +from typing import Optional from workflow_rerun.log_collector import collect_logs_for_run def record_rerun_to_db(repository_full_name: str, run_id: int, ticket_number: int, rerunner_run_id: int, error_text: str): @@ -68,7 +69,7 @@ def rerun_failed_jobs(repository_name: str, run_id: int, session: requests.Sessi LOGGER.info(f'RUN RETRIGGERED SUCCESSFULLY: {run.html_url}') def analyze_and_rerun(run, repository_name: str, run_id: int, rerunner_run_id: int, - errors_file: Path, is_dry_run: bool, session: requests.Session): + errors_file: Path, patterns_dir: Optional[Path], is_dry_run: bool, session: requests.Session): with tempfile.TemporaryDirectory() as temp_dir: logs_dir = Path(temp_dir) collect_logs_for_run( @@ -79,7 +80,8 @@ def analyze_and_rerun(run, repository_name: str, run_id: int, rerunner_run_id: i log_analyzer = LogAnalyzer( path_to_logs=logs_dir, - path_to_errors_file=errors_file + path_to_errors_file=errors_file, + patterns_dir=patterns_dir ) log_analyzer.analyze() @@ -108,6 +110,7 @@ def analyze_and_rerun(run, repository_name: str, run_id: int, rerunner_run_id: i rerunner_run_id = args.rerunner_run_id repository_name = args.repository_name errors_file = args.errors_to_look_for_file + patterns_dir = args.patterns_dir is_dry_run = args.dry_run if is_dry_run: LOGGER.info('RUNNING IN DRY RUN MODE. IF ERROR WILL BE FOUND, WILL NOT RETRIGGER') @@ -133,4 +136,4 @@ def analyze_and_rerun(run, repository_name: str, run_id: int, rerunner_run_id: i LOGGER.info(f'THERE ARE {run.run_attempt} ATTEMPTS ALREADY. NOT CHECKING LOGS AND NOT RETRIGGERING. EXITING') sys.exit(0) - analyze_and_rerun(run, repository_name, run_id, rerunner_run_id, errors_file, is_dry_run, session) + analyze_and_rerun(run, repository_name, run_id, rerunner_run_id, errors_file, patterns_dir, is_dry_run, session) diff --git a/.github/scripts/workflow_rerun/tests/data/empty_errors.json b/.github/scripts/workflow_rerun/tests/data/empty_errors.json new file mode 100644 index 000000000000..0637a088a01e --- /dev/null +++ b/.github/scripts/workflow_rerun/tests/data/empty_errors.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.github/scripts/workflow_rerun/tests/data/patterns/aaaa1111bbbb2222.json b/.github/scripts/workflow_rerun/tests/data/patterns/aaaa1111bbbb2222.json new file mode 100644 index 000000000000..9f89b3d52830 --- /dev/null +++ b/.github/scripts/workflow_rerun/tests/data/patterns/aaaa1111bbbb2222.json @@ -0,0 +1,18 @@ +{ + "schema_version": "1.0", + "signature": "label empty or too long|Flaky Test", + "signature_hash": "aaaa1111bbbb2222", + "title": "Sample transient failure used for rerunner pattern tests", + "category": "Flaky Test", + "count": 2, + "first_seen": "2026-07-01T10:00:00Z", + "last_seen": "2026-07-13T10:00:00Z", + "recent_run_urls": [ + "https://github.com/openvinotoolkit/openvino/actions/runs/29268143021" + ], + "affected_prs": [], + "recent_timestamps": [ + "2026-07-13T10:00:00Z" + ], + "rerun_search_string": "label empty or too long" +} \ No newline at end of file diff --git a/.github/scripts/workflow_rerun/tests/data/patterns/cccc3333dddd4444.json b/.github/scripts/workflow_rerun/tests/data/patterns/cccc3333dddd4444.json new file mode 100644 index 000000000000..b135e6c434fd --- /dev/null +++ b/.github/scripts/workflow_rerun/tests/data/patterns/cccc3333dddd4444.json @@ -0,0 +1,18 @@ +{ + "schema_version": "1.0", + "signature": "assertion failed in unit test|Code Issue", + "signature_hash": "cccc3333dddd4444", + "title": "Deterministic failure that must never trigger a rerun", + "category": "Code Issue", + "count": 1, + "first_seen": "2026-07-10T10:00:00Z", + "last_seen": "2026-07-10T10:00:00Z", + "recent_run_urls": [ + "https://github.com/openvinotoolkit/openvino/actions/runs/29268143022" + ], + "affected_prs": [], + "recent_timestamps": [ + "2026-07-10T10:00:00Z" + ], + "rerun_search_string": null +} \ No newline at end of file diff --git a/.github/scripts/workflow_rerun/tests/gh_test_utils.py b/.github/scripts/workflow_rerun/tests/gh_test_utils.py new file mode 100644 index 000000000000..7e49fb75a70d --- /dev/null +++ b/.github/scripts/workflow_rerun/tests/gh_test_utils.py @@ -0,0 +1,28 @@ +# Copyright (C) 2018-2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +""" +Shared helpers for live-GitHub integration tests +""" + +from typing import Optional + +from github.Repository import Repository +from github.WorkflowRun import WorkflowRun + + +def find_failed_run_with_logs(gh_repo: Repository, + created_filter: str, + max_candidates: int = 20) -> Optional[WorkflowRun]: + """ + Some workflow runs reported as "failure" never actually start any jobs + (e.g. the workflow file itself is invalid), so they have no logs to + download/analyze. This looks through the most recent failed runs and + returns the first one that has at least one failed job, or None if no + such run is found among the checked candidates. + """ + candidate_runs = gh_repo.get_workflow_runs(status='failure', created=created_filter) + for candidate_run in candidate_runs[:max_candidates]: + if any(job.conclusion == 'failure' for job in candidate_run.jobs()): + return candidate_run + return None diff --git a/.github/scripts/workflow_rerun/tests/integration_test.py b/.github/scripts/workflow_rerun/tests/integration_test.py index 90ea74f111a1..e8738620939e 100644 --- a/.github/scripts/workflow_rerun/tests/integration_test.py +++ b/.github/scripts/workflow_rerun/tests/integration_test.py @@ -22,6 +22,8 @@ from workflow_rerun.log_collector import collect_logs_for_run from workflow_rerun.rerunner import analyze_and_rerun +from workflow_rerun.tests.gh_test_utils import find_failed_run_with_logs + class IntegrationTest(unittest.TestCase): """ @@ -54,8 +56,10 @@ def setUpClass(cls) -> None: gh_repo = cls.github.get_repo(full_name_or_id='openvinotoolkit/openvino') oldest_allowed_date = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d') - cls.wf_run = gh_repo.get_workflow_runs(status='failure', - created=f">={oldest_allowed_date}")[0] + cls.wf_run = find_failed_run_with_logs(gh_repo=gh_repo, + created_filter=f">={oldest_allowed_date}", + max_candidates=50) + print(f'Workflow run for testing: {cls.wf_run}', flush=True) def setUp(self): @@ -113,6 +117,7 @@ def fake_collect_logs_for_run(*, run, logs_dir: Path, session): run_id=run_id, rerunner_run_id=rerunner_run_id, errors_file=self.errors_to_look_for_file, + patterns_dir=None, is_dry_run=False, session=mock_session ) diff --git a/.github/scripts/workflow_rerun/tests/log_analyzer_test.py b/.github/scripts/workflow_rerun/tests/log_analyzer_test.py index 5204bf2722f4..908557759b94 100644 --- a/.github/scripts/workflow_rerun/tests/log_analyzer_test.py +++ b/.github/scripts/workflow_rerun/tests/log_analyzer_test.py @@ -9,7 +9,7 @@ from pathlib import Path -from workflow_rerun.log_analyzer import LogAnalyzer +from workflow_rerun.log_analyzer import CI_DOCTOR_PATTERN_TICKET, LogAnalyzer class LogAnalyzerTest(unittest.TestCase): @@ -29,6 +29,10 @@ def setUp(self) -> None: self.errors_to_look_for_file = self._cwd.parent.joinpath( 'errors_to_look_for.json' ) + self.empty_errors_file = self._cwd.joinpath('data').joinpath( + 'empty_errors.json' + ) + self.patterns_dir = self._cwd.joinpath('data').joinpath('patterns') def test_log_analyzer_instantiation(self) -> None: """ @@ -105,3 +109,57 @@ def test_analyzer_wo_error(self) -> None: ) analyzer.analyze() self.assertFalse(analyzer.found_matching_error) + + def test_patterns_loaded_from_directory(self) -> None: + """ + Ensure LogAnalyzer loads rerun_search_string values from CI Doctor MQ + pattern files (tagged with the CI_DOCTOR_PATTERN_TICKET sentinel ticket) + and skips patterns whose rerun_search_string is null. + """ + analyzer = LogAnalyzer( + path_to_logs=self.logs_dir_wo_error, + path_to_errors_file=self.empty_errors_file, + patterns_dir=self.patterns_dir, + ) + + pattern_errors = analyzer._errors_to_look_for + self.assertEqual( + len(pattern_errors), + 1, + 'Only the pattern with a non-null rerun_search_string should be loaded', + ) + self.assertEqual(pattern_errors[0]['error_text'], 'label empty or too long') + self.assertEqual( + pattern_errors[0]['ticket'], + CI_DOCTOR_PATTERN_TICKET, + 'Pattern-derived errors must carry the CI_DOCTOR_PATTERN_TICKET sentinel', + ) + + def test_analyzer_matches_pattern_search_string(self) -> None: + """ + Ensure LogAnalyzer can find an error using a rerun_search_string coming + from a CI Doctor MQ pattern file, reporting the sentinel ticket for it. + """ + analyzer = LogAnalyzer( + path_to_logs=self.logs_dir_with_error, + path_to_errors_file=self.empty_errors_file, + patterns_dir=self.patterns_dir, + ) + analyzer.analyze() + self.assertTrue(analyzer.found_matching_error) + self.assertEqual(analyzer.found_error_ticket, CI_DOCTOR_PATTERN_TICKET) + self.assertEqual(analyzer.matched_error_text, 'label empty or too long') + + def test_missing_patterns_dir_is_ignored(self) -> None: + """ + Ensure a non-existent patterns_dir does not break analysis and simply + adds no pattern-derived errors. + """ + analyzer = LogAnalyzer( + path_to_logs=self.logs_dir_wo_error, + path_to_errors_file=self.empty_errors_file, + patterns_dir=self._cwd.joinpath('data').joinpath('does_not_exist'), + ) + self.assertEqual(analyzer._errors_to_look_for, []) + analyzer.analyze() + self.assertFalse(analyzer.found_matching_error) diff --git a/.github/scripts/workflow_rerun/tests/log_collector_test.py b/.github/scripts/workflow_rerun/tests/log_collector_test.py index e818c1e270d3..82b0c26c5bda 100644 --- a/.github/scripts/workflow_rerun/tests/log_collector_test.py +++ b/.github/scripts/workflow_rerun/tests/log_collector_test.py @@ -17,6 +17,7 @@ from requests.packages.urllib3.util.retry import Retry from workflow_rerun.log_collector import collect_logs_for_run +from workflow_rerun.tests.gh_test_utils import find_failed_run_with_logs class LogCollectorTest(unittest.TestCase): @@ -46,8 +47,9 @@ def setUpClass(cls) -> None: created=f">={oldest_allowed_date}")[0] print(f'Successful workflow run for testing: {cls.successful_workflow_run}', flush=True) - cls.failed_workflow_run = gh_repo.get_workflow_runs(status='failure', - created=f">={oldest_allowed_date}")[0] + cls.failed_workflow_run = find_failed_run_with_logs(gh_repo=gh_repo, + created_filter=f">={oldest_allowed_date}", + max_candidates=50) print(f'Failed workflow run for testing: {cls.failed_workflow_run}', flush=True) def setUp(self): diff --git a/.github/workflows/ci-doctor-mq.lock.yml b/.github/workflows/ci-doctor-mq.lock.yml index 27bd892df144..2967e542f528 100644 --- a/.github/workflows/ci-doctor-mq.lock.yml +++ b/.github/workflows/ci-doctor-mq.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"92a2828636c9ba8348f197a5ddedf84fe5ee9a1f7de0fef9717a2c05ad519a7e","body_hash":"d27390144e24c8796d074263a2b421cfc68ae47aa961accad44583e02977c2e9","compiler_version":"v0.81.6","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6","engine_versions":{"copilot":"1.0.65"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"92a2828636c9ba8348f197a5ddedf84fe5ee9a1f7de0fef9717a2c05ad519a7e","body_hash":"ed50e33a972a2ccd6b3b070a40388667d0cd8f288a2685960efc9aa39bda9394","compiler_version":"v0.81.6","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6","engine_versions":{"copilot":"1.0.65"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN","TEAMS_WEBHOOK_URL"],"actions":[{"repo":"actions/cache/restore","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/cache/save","sha":"27d5ce7f107fe9357f9df03efb73ab90386fccae","version":"v5.0.5"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"ba6380cc6e5be5d21677bebe04d52fb48e3abec7","version":"v0.81.6"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11","digest":"sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.11@sha256:979723c628182da7729333f2208bb249fd25ddee579645cf9a3892d681a929c7"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11","digest":"sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.11@sha256:807e4831999b44513b0a66e5859d478dc4da7ae74ab1918cec967d513f95bf9d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11","digest":"sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.11@sha256:ff27ea0525ad953a6adee28a5fbe9d2e22be47dbec755c15767af4ea3f91df7d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.30","digest":"sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.30@sha256:35625d1a2269b1238606078c879f59a91cffc4ac33eb54bf39c6418822c1a8be"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.4.0","digest":"sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036","pinned_image":"ghcr.io/github/github-mcp-server:v1.4.0@sha256:2afb26356481d1a350e14544a6e160f7f7ec1561a1ea309b823665abf0309036"}]} # This file was automatically generated by gh-aw (v0.81.6). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/ci-doctor-mq.md b/.github/workflows/ci-doctor-mq.md index 5c4169e4438c..d49b47c8c36e 100644 --- a/.github/workflows/ci-doctor-mq.md +++ b/.github/workflows/ci-doctor-mq.md @@ -404,6 +404,7 @@ Rules that apply to all artefact types: → deduplicate by URL, then truncate to first 10 entries record.recent_timestamps = [NOW] + existing.recent_timestamps → keep only entries where timestamp >= (NOW - 24 hours) + record.rerun_search_string = RERUN_SEARCH_STRING ← recompute per Step C.1 (refresh always) ELSE: record.schema_version = "1.0" record.signature = the | signature string from Step A @@ -416,8 +417,17 @@ Rules that apply to all artefact types: record.recent_run_urls = [CURRENT_RUN_URL] record.affected_prs = (if CURRENT_PR_URL: [CURRENT_PR_URL] else []) record.recent_timestamps = [NOW] + record.rerun_search_string = RERUN_SEARCH_STRING ← compute per Step C.1 ~~~ + **Step C.1 — Compute and verify `RERUN_SEARCH_STRING` (the workflow_rerunner hook):** + This field feeds the automated rerunner (`.github/scripts/workflow_rerun`), which greps the failed run's logs for this string and, on a match, re-runs the failed jobs — exactly like the static entries in `.github/scripts/workflow_rerun/errors_to_look_for.json`. A rerun only helps for **transient** failures, and a string that never matches is useless: + - IF `category` is NOT one of `Flaky Test`, `Infrastructure`, `Network`, or `External Service` (i.e. a deterministic `Code Issue`/`Dependencies`/`Configuration` failure a restart cannot fix): set `RERUN_SEARCH_STRING = null`. A search string here would make the rerunner loop on an unfixable failure. + - ELSE: set `RERUN_SEARCH_STRING` to a **short, stable substring taken verbatim from a single raw log line** that uniquely identifies this failure (e.g., `Could not resolve host`, `Connection reset by peer`, `runner has received a shutdown signal`). Strip everything volatile that changes between runs or across jobs: absolute paths, line/column numbers, hex addresses, PIDs, timestamps, run IDs, commit SHAs, tmp dirs, UUIDs, and embedded job/runner/OS/shard names. + - Always recompute on update so a category change (e.g. `Flaky Test` → `Code Issue`) correctly clears or sets the string. + + **Verify a non-null value against the rerunner's exact matcher** ([`log_analyzer.py`](../scripts/workflow_rerun/log_analyzer.py)): it normalizes a string by replacing each run of non-alphanumeric chars (`[^A-Za-z0-9]+`) with one space, lower-casing, and stripping (so `Could not resolve host: github.com` → `could not resolve host github com`), then checks whether the normalized string is a plain substring of a **single** normalized log line (matches never span line breaks). Confirm the normalized `RERUN_SEARCH_STRING` is a substring of a normalized real log line from the pre-downloaded logs, is non-empty after normalization, and is specific enough to avoid matching benign lines (no bare `error`/`failed`/`warning`). If you cannot satisfy this, set `RERUN_SEARCH_STRING = null` rather than store an unverified guess. + **Step D — Write the file:** Write `record` as JSON to `/tmp/gh-aw/repo-memory/default/mq/patterns/.json`. Overwrite the file completely with the new content. @@ -429,6 +439,8 @@ Rules that apply to all artefact types: - `recent_timestamps` contains the current timestamp `NOW` as the first entry - `last_seen` equals `NOW` - `first_seen` has NOT changed from `existing.first_seen` (if file existed before) + - `rerun_search_string` is present and is either a non-empty string (only for the transient categories `Flaky Test`/`Infrastructure`/`Network`/`External Service`) or `null` (for every other category) + - if `rerun_search_string` is non-null, it still satisfies the Step C.1 verification (normalized substring of a single actual failure-log line, non-empty after normalization); otherwise set it to `null` If any check fails, you have a bug in your write logic. Fix it before proceeding. @@ -450,6 +462,22 @@ Rules that apply to all artefact types: 4. **Save Artifacts**: Store detailed logs and analysis in the cached directories. +5. **Backfill Missing `rerun_search_string` Fields — MANDATORY sweep, every run:** + + Some pattern files were written before `rerun_search_string` existed and are missing the field entirely (as opposed to having it explicitly set to `null`). Every time you run Phase 5, after step 2 has written/updated the current failure's pattern file, sweep **all** other files in `/tmp/gh-aw/repo-memory/default/mq/patterns/` and backfill any that lack the key: + + - For each `.json` file in the patterns directory (including ones untouched by this run): + 1. Read and parse the file. + 2. If the `rerun_search_string` key is **already present** (even if its value is `null`), skip it — do not touch a file that already conforms to the schema. + 3. If the key is **missing**, derive it from the file's own `signature` field (`|`) using the same rules as Step C.1, without needing the original failure logs: + - Split `signature` on the last `|` to recover `` and ``. + - IF `` is NOT one of `Flaky Test`, `Infrastructure`, `Network`, or `External Service`: set `rerun_search_string = null`. + - ELSE: set `rerun_search_string = ` verbatim (the signature's error component was already stripped of volatile tokens per Step A, so no further normalization is needed). Since the original raw logs are unlikely to still be available for a backfill, you do NOT need to re-verify it against a live log line as Step C.1 otherwise requires — only confirm it is non-empty and not just generic noise (e.g., not solely `error`, `failed`, or `warning`). If it fails that minimal check, set it to `null` instead of guessing. + 4. Write the updated object back to the same file, changing only the `rerun_search_string` key — do not alter `count`, timestamps, `signature`, `signature_hash`, or any other field. + 5. Validate the rewritten file against `pattern.schema.json` (see the validation procedure in the "Artefact schemas" block) and confirm `rerun_search_string` is now present as either a non-empty string or `null`. + + This step must never modify a file's `count`, `first_seen`, `last_seen`, or timestamp arrays — its only job is adding the missing key. + ### Phase 5.5: Recurring Failure Escalation Check After updating the pattern database (Phase 5 step 2), check whether the current failure's pattern has occurred **3 or more times in the last 12 hours**. diff --git a/.github/workflows/workflow_rerunner.yml b/.github/workflows/workflow_rerunner.yml index a2d816d2e47a..d9cc0a07edd0 100644 --- a/.github/workflows/workflow_rerunner.yml +++ b/.github/workflows/workflow_rerunner.yml @@ -47,6 +47,15 @@ jobs: with: sparse-checkout: '.github/scripts/workflow_rerun' + - name: Checkout CI Doctor MQ patterns + uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries + timeout-minutes: 15 + continue-on-error: true + with: + ref: memory/ci-doctor-mq + sparse-checkout: 'mq/patterns' + path: ci-doctor-patterns + - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -74,7 +83,8 @@ jobs: python3 ${{ github.workspace }}/.github/scripts/workflow_rerun/rerunner.py \ --run-id ${{ github.event.workflow_run.id }} \ --rerunner-run-id ${{ github.run_id }} \ - --repository-name ${GITHUB_REPOSITORY} + --repository-name ${GITHUB_REPOSITORY} \ + --patterns-dir ${{ github.workspace }}/ci-doctor-patterns/mq/patterns rerunner_tests: name: Rerunner Tests @@ -88,6 +98,16 @@ jobs: sparse-checkout: '.github/scripts/workflow_rerun' lfs: false + - name: Checkout CI Doctor MQ patterns + uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries + timeout-minutes: 15 + continue-on-error: true + with: + repository: openvinotoolkit/openvino + ref: memory/ci-doctor-mq + sparse-checkout: 'mq/patterns' + path: ci-doctor-patterns + - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -117,3 +137,6 @@ jobs: run_id=$(python3 -c "from github import Github, Auth; import os; from datetime import datetime, timedelta, timezone; github=Github(auth=Auth.Token(token=os.environ.get('GITHUB_TOKEN'))); repo = github.get_repo('${GITHUB_REPOSITORY}'); cutoff_date=(datetime.now(timezone.utc)-timedelta(days=14)).date().isoformat(); runs=repo.get_workflow_runs(status='success', created=f'>={cutoff_date}'); print(runs[0].id)") python3 rerunner.py --repository-name ${GITHUB_REPOSITORY} --run-id $run_id --rerunner-run-id ${{ github.run_id }} --dry-run + + # Same as above, but additionally exercising the CI Doctor MQ patterns integration + python3 rerunner.py --repository-name ${GITHUB_REPOSITORY} --run-id $run_id --rerunner-run-id ${{ github.run_id }} --patterns-dir ${{ github.workspace }}/ci-doctor-patterns/mq/patterns --dry-run