Multiple fixes

.github/workflows/pr_check.yml

@@ -0,0 +1,17 @@
+name: PR Pre Checks
+
+on:
+  pull_request:
+  push:
+    branches:
+      - mainline
+
+jobs:
+  pre-commit-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          pre-commit install
+          pre-commit run --all-files

.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+
+# Ruff stuff:
+.ruff_cache/
+
+venv/
+env/
+.venv/
+.env/
+
+*.egg-info/

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.14.0
+  hooks:
+    # Run the linter.
+    - id: ruff-check
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+
+exclude: |
+  (?x)^(
+  release_config.py |
+  rolling-release-update.py |
+  run_interdiff.py |
+  )$

ciq-cherry-pick.py

@@ -1,44 +1,50 @@
 import argparse
+import logging
 import os
+import re
 import subprocess
+
 import git
-from ciq_helpers import CIQ_cherry_pick_commit_standardization
-from ciq_helpers import CIQ_original_commit_author_to_tag_string
-# from ciq_helpers import *
 
-MERGE_MSG = git.Repo(os.getcwd()).git_dir + '/MERGE_MSG'
+from ciq_helpers import (
+    CIQ_cherry_pick_commit_standardization,
+    CIQ_commit_exists_in_current_branch,
+    CIQ_find_fixes_in_mainline_current_branch,
+    CIQ_fixes_references,
+    CIQ_get_full_hash,
+    CIQ_original_commit_author_to_tag_string,
+)
 
-if __name__ == '__main__':
-    print("CIQ custom cherry picker")
-    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument('--sha', help='Target SHA1 to cherry-pick')
-    parser.add_argument('--ticket', help='Ticket associated to cherry-pick work, comma separated list is supported.')
-    parser.add_argument('--ciq-tag', help="Tags for commit message <feature><-optional modifier> <identifier>.\n"
-                        "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n"
-                        "         cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n"
-                        "         cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n"
-                        "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n")
-    args = parser.parse_args()
+MERGE_MSG = git.Repo(os.getcwd()).git_dir + "/MERGE_MSG"
+
+
+def check_fixes(sha):
+    """
+    Checks if commit has "Fixes:" references and if so, it checks if the
+    commit(s) that it tries to fix are part of the current branch
+    """
+
+    fixes = CIQ_fixes_references(repo_path=".", sha=sha)
+    if len(fixes) == 0:
+        logging.warning("The commit you try to cherry pick has no Fixes: reference; review it carefully")
+        return
+
+    for fix in fixes:
+        if not CIQ_commit_exists_in_current_branch(".", fix):
+            raise RuntimeError(f"The commit you want to cherry pick references a Fixes {fix}: but this is not here")
 
+
+def cherry_pick(sha, ciq_tags, jira_ticket):
     # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression
-    git_sha_res = subprocess.run(['git', 'show', '--pretty=%H', '-s', args.sha], stdout=subprocess.PIPE)
-    if git_sha_res.returncode != 0:
-        print(f"[FAILED] git show --pretty=%H -s {args.sha}")
-        print("Subprocess Call:")
-        print(git_sha_res)
-        print("")
-    else:
-        args.sha = git_sha_res.stdout.decode('utf-8').strip()
+    full_sha = CIQ_get_full_hash(".", sha)
 
-    tags = []
-    if args.ciq_tag is not None:
-        tags = args.ciq_tag.split(',')
+    check_fixes(full_sha)
 
-    author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha)
-    if author is None:
+    author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=full_sha)
+    if author is None:  # TODO raise Exception maybe
         exit(1)
 
-    git_res = subprocess.run(['git', 'cherry-pick', '-nsx', args.sha])
+    git_res = subprocess.run(["git", "cherry-pick", "-nsx", full_sha])  # Move it into a separate method
     if git_res.returncode != 0:
         print(f"[FAILED] git cherry-pick -nsx {args.sha}")
         print("       Manually resolve conflict and include `upstream-diff` tag in commit message")
@@ -47,22 +53,68 @@
         print("")
 
     print(os.getcwd())
-    subprocess.run(['cp', MERGE_MSG, f'{MERGE_MSG}.bak'])
+    subprocess.run(["cp", MERGE_MSG, f"{MERGE_MSG}.bak"])
 
     tags.append(author)
 
     with open(MERGE_MSG, "r") as file:
         original_msg = file.readlines()
 
-    new_msg = CIQ_cherry_pick_commit_standardization(original_msg, args.sha, jira=args.ticket, tags=tags)
+    new_msg = CIQ_cherry_pick_commit_standardization(original_msg, full_sha, jira=jira_ticket, tags=ciq_tags)
 
     print(f"Cherry Pick New Message for {args.sha}")
-    for line in new_msg:
-        print(line.strip('\n'))
     print(f"\n Original Message located here: {MERGE_MSG}.bak")
 
     with open(MERGE_MSG, "w") as file:
         file.writelines(new_msg)
 
     if git_res.returncode == 0:
-        subprocess.run(['git', 'commit', '-F', MERGE_MSG])
+        subprocess.run(["git", "commit", "-F", MERGE_MSG])
+
+
+def cherry_pick_fixes(sha, ciq_tags, jira_ticket, upstream_ref):
+    fixes_in_mainline = CIQ_find_fixes_in_mainline_current_branch(".", upstream_ref, sha)
+
+    # Replace cve with cve-bf
+    # Leave cve-pre and cve-bf as they are
+    ciq_tags = [re.sub(r"^cve ", "cve-bf ", s) for s in ciq_tags]
+    for full_hash, display_str in fixes_in_mainline:
+        print(f"Extra cherry picking {display_str}")
+        full_cherry_pick(sha=full_hash, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref)
+
+
+def full_cherry_pick(sha, ciq_tags, jira_ticket, upstream_ref):
+    # Cherry pick the commit
+    cherry_pick(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket)
+
+    # Cherry pick the fixed-by dependencies
+    cherry_pick_fixes(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket, upstream_ref=upstream_ref)
+
+
+if __name__ == "__main__":
+    print("CIQ custom cherry picker")
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("--sha", help="Target SHA1 to cherry-pick")
+    parser.add_argument("--ticket", help="Ticket associated to cherry-pick work, comma separated list is supported.")
+    parser.add_argument(
+        "--ciq-tag",
+        help="Tags for commit message <feature><-optional modifier> <identifier>.\n"
+        "example: cve CVE-2022-45884 - A patch for a CVE Fix.\n"
+        "         cve-bf CVE-1974-0001 - A bug fix for a CVE currently being patched\n"
+        "         cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n"
+        "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n",
+    )
+
+    parser.add_argument(
+        "--upstream-ref",
+        default="origin/kernel-mainline",
+        help="Reference to upstream mainline branch (default: origin/kernel-mainline)",
+    )
+
+    args = parser.parse_args()
+
+    tags = []
+    if args.ciq_tag is not None:
+        tags = args.ciq_tag.split(",")
+
+    full_cherry_pick(sha=args.sha, ciq_tags=tags, jira_ticket=args.ticket, upstream_ref=args.upstream_ref)

ciq_helpers.py

@@ -3,11 +3,12 @@
 
 # CIQ Kernel Tools  function library
 
-import git
 import os
 import re
 import subprocess
 
+import git
+
 
 def process_full_commit_message(commit):
     """Process the full git commit message specific to the CIQ Kernel Tools.
@@ -84,9 +85,7 @@ def get_backport_commit_data(repo, branch, common_ancestor, allow_duplicates=Fal
 
     for line in lines:
         if len(commit) > 0 and line.startswith(b"commit "):
-            upstream_commit, cves, tickets, upstream_subject, repo_commit = (
-                process_full_commit_message(commit)
-            )
+            upstream_commit, cves, tickets, upstream_subject, repo_commit = process_full_commit_message(commit)
             if upstream_commit in upstream_commits:
                 print(f"WARNING: {upstream_commit} already in upstream_commits")
                 if not allow_duplicates:
@@ -104,9 +103,7 @@ def get_backport_commit_data(repo, branch, common_ancestor, allow_duplicates=Fal
     return upstream_commits, True
 
 
-def CIQ_cherry_pick_commit_standardization(
-    lines, commit, tags=None, jira="", optional_msg=""
-):
+def CIQ_cherry_pick_commit_standardization(lines, commit, tags=None, jira="", optional_msg=""):
     """Standardize CIQ the cherry-pick commit message.
     Parameters:
     lines: Original SHAS commit message.
@@ -159,13 +156,154 @@ def CIQ_original_commit_author_to_tag_string(repo_path, sha):
 
     Return: String for Tag
     """
-    git_auth_res = subprocess.run(['git', 'show', '--pretty="%aN <%aE>"', '--no-patch', sha], stderr=subprocess.PIPE,
-                                  stdout=subprocess.PIPE, cwd=repo_path)
+    git_auth_res = subprocess.run(
+        ["git", "show", '--pretty="%aN <%aE>"', "--no-patch", sha],
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        cwd=repo_path,
+    )
     if git_auth_res.returncode != 0:
         print(f"[FAILED] git show --pretty='%aN <%aE>' --no-patch {sha}")
         print(f"[FAILED][STDERR:{git_auth_res.returncode}] {git_auth_res.stderr.decode('utf-8')}")
         return None
-    return "commit-author " + git_auth_res.stdout.decode('utf-8').replace('\"', '').strip()
+    return "commit-author " + git_auth_res.stdout.decode("utf-8").replace('"', "").strip()
+
+
+def CIQ_run_git(repo_path, args):
+    """
+    Run a git command in the given repository and return its output as a string.
+    """
+    result = subprocess.run(["git", "-C", repo_path] + args, text=True, capture_output=True, check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}")
+
+    return result.stdout
+
+
+def CIQ_get_commit_body(repo_path, sha):
+    return CIQ_run_git(repo_path, ["show", "-s", sha, "--format=%B"])
+
+
+def CIQ_extract_fixes_references_from_commit_body_lines(lines):
+    fixes = []
+    for line in lines:
+        m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE)
+        if not m:
+            continue
+
+        fixes.append(m.group(1))
+
+    return fixes
+
+
+def CIQ_fixes_references(repo_path, sha):
+    """
+    If commit message of sha contains lines like
+    Fixes: <short_fixed>, this returns a list of <short_fixed>, otherwise an empty list
+    """
+
+    commit_body = CIQ_get_commit_body(repo_path, sha)
+    return CIQ_extract_fixes_references_from_commit_body_lines(lines=commit_body.splitlines())
+
+
+def CIQ_get_full_hash(repo, short_hash):
+    return CIQ_run_git(repo, ["show", "-s", "--pretty=%H", short_hash]).strip()
+
+
+def CIQ_get_current_branch(repo):
+    return CIQ_run_git(repo, ["branch", "--show-current"]).strip()
+
+
+def CIQ_hash_exists_in_ref(repo, pr_ref, hash_):
+    """
+    Return True is hash_ is reachable from pr_branch
+    """
+
+    try:
+        CIQ_run_git(repo, ["merge-base", "--is-ancestor", hash_, pr_ref])
+        return True
+    except RuntimeError:
+        return False
+
+
+# TODO think of a better name
+def CIQ_commit_exists_in_branch(repo, pr_branch, upstream_hash_):
+    """
+    Return True if upstream_hash_ has been backported and it exists in the pr branch
+    """
+
+    # First check if the commit has been backported by CIQ
+    output = CIQ_run_git(repo, ["log", pr_branch, "--grep", "commit " + upstream_hash_])
+    if output:
+        return True
+
+    # If it was not backported by CIQ, maybe it came from upstream as it is
+    return CIQ_hash_exists_in_ref(repo, pr_branch, upstream_hash_)
+
+
+def CIQ_commit_exists_in_current_branch(repo, upstream_hash_):
+    """
+    Return True if upstream_hash_ has been backported and it exists in the current branch
+    """
+
+    current_branch = CIQ_get_current_branch(repo)
+    full_upstream_hash = CIQ_get_full_hash(repo, upstream_hash_)
+
+    return CIQ_commit_exists_in_branch(repo, current_branch, full_upstream_hash)
+
+
+def CIQ_find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_):
+    """
+    Return unique commits in upstream_ref that have Fixes: <N chars of hash_> in their message, case-insensitive,
+    if they have not been commited in the pr_branch.
+    Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length.
+    Returns a list of tuples: (full_hash, display_string)
+    """
+    results = []
+
+    # Prepare hash prefixes from 12 down to 6
+    hash_prefixes = [hash_[:index] for index in range(12, 5, -1)]
+
+    # Get all commits with 'Fixes:' in the message
+    output = CIQ_run_git(
+        repo,
+        [
+            "log",
+            upstream_ref,
+            "--grep",
+            "Fixes:",
+            "-i",
+            "--format=%H %h %s (%an)%x0a%B%x00",
+        ],
+    ).strip()
+    if not output:
+        return []
+
+    # Each commit is separated by a NUL character and a newline
+    commits = output.split("\x00\x0a")
+    for commit in commits:
+        if not commit.strip():
+            continue
+
+        lines = commit.splitlines()
+        # The first line is the summary, the rest is the body
+        header = lines[0]
+        full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split())
+        fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:])
+        for fix in fixes:
+            for prefix in hash_prefixes:
+                if fix.lower().startswith(prefix.lower()):
+                    if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash):
+                        results.append((full_hash, display_string))
+                    break
+
+    return results
+
+
+def CIQ_find_fixes_in_mainline_current_branch(repo, upstream_ref, hash_):
+    current_branch = CIQ_get_current_branch(repo)
+
+    return CIQ_find_fixes_in_mainline(repo, current_branch, upstream_ref, hash_)
 
 
 def repo_init(repo):