Fix script to handle SARIF file recategorization

adding GitPython lib

cleaning the files for redundant

made Codeql executable with bazel

fixing bzel run issues

adding a version

removing timeout

addresses several issues related to environment-specific paths and execution context

removing the ref

Author: Saumya-R

.github/codeql/codeql-config.yml

@@ -19,3 +19,11 @@ paths-ignore:
   - "**/test/**"
   - "**/mock/**"
   - "**/codeql-coding-standards-repo/**"
+  - "**/examples/**"
+  - "**/docs/**"
+  - "**/target/**"
+  - "**/bazel-*/**"
+  - "**/.git/**"
+  - "**/node_modules/**"
+  - "**/build/**"
+  - "**/dist/**"

.github/workflows/codeql-multiple-repo-scan.yml

@@ -32,6 +32,7 @@ jobs:
   analyze-repos:
     name: Analyze Multiple Repositories
     runs-on: ubuntu-latest
+    timeout-minutes: 120 # Prevent indefinite hanging
     permissions:
       security-events: write
       packages: read
@@ -40,25 +41,23 @@ jobs:
     steps:
       - name: Checkout central repository
         uses: actions/checkout@v4
-      - name: Checkout CodeQL Coding Standards scripts
-        uses: actions/checkout@v4
-        with:
-          repository: github/codeql-coding-standards
-          path: codeql-coding-standards-repo # Klonen in diesen Ordner
-          ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev'
-      # Add coding standard packages and dependencies
-      - name: Install Python dependencies for Coding Standards scripts
+      - name: Install sarif-tools for HTML report generation
         run: |
           python3 -m pip install --upgrade pip
-          pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools
-      - name: Parse known_good.json and create repos.json
-        id: parse-repos
-        run: |
-          scripts/workflow/parse_repos.sh
+          pip3 install sarif-tools
+      - name: Cache repository checkouts
+        uses: actions/cache@v4
+        with:
+          path: repos/
+          key: repos-${{ hashFiles('known_good.json') }}
+          restore-keys: |
+            repos-
       - name: Checkout all pinned repositories
         id: checkout-repos
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          scripts/workflow/checkout_repos.sh
+          bazel run //scripts/tooling:checkout_repos
       - name: Initialize CodeQL for all repositories
         uses: github/codeql-action/init@v4
         with:
@@ -69,16 +68,30 @@ jobs:
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v4
         with:
-          upload-database: false # Don't upload databases for each repo
+          upload-database: false
           output: sarif-results/
           category: "multi-repo-scan"
+      - name: Cleanup repository checkouts
+        if: always()
+        run: |
+          echo "Cleaning up checked out repositories to free disk space"
+          rm -rf repos/
+      - name: Checkout CodeQL Coding Standards scripts
+        uses: actions/checkout@v4
+        with:
+          repository: github/codeql-coding-standards
+          path: codeql-coding-standards-repo
       - name: Recategorize Guidelines
         if: always()
         run: |
-          scripts/workflow/recategorize_guidelines.sh
+          bazel run //scripts/tooling:recategorize_guidelines
       - name: Generate HTML Report from SARIF
         run: |
           SARIF_FILE="sarif-results/cpp.sarif"
+          if [ ! -f "$SARIF_FILE" ]; then
+            echo "Error: SARIF file not found at $SARIF_FILE"
+            exit 1
+          fi
           sarif html "$SARIF_FILE" --output codeql-report.html
       - name: Upload SARIF results as artifact
         uses: actions/upload-artifact@v4

feature_integration_tests/test_cases/BUILD

@@ -25,6 +25,9 @@ compile_pip_requirements(
         "requirements.txt",
         "@score_tooling//python_basics:requirements.txt",
     ],
+    extra_args = [
+        "--no-annotate",
+    ],
     requirements_txt = "requirements.txt.lock",
     tags = [
         "manual",

feature_integration_tests/test_cases/requirements.txt.lock

@@ -2,22 +2,17 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    bazel run //feature_integration_tests/python_test_cases:requirements.update
+#    bazel run //feature_integration_tests/test_cases:requirements.update
 #
-basedpyright==1.29.2 \
-    --hash=sha256:12c49186003b9f69a028615da883ef97035ea2119a9e3f93a00091b3a27088a6 \
-    --hash=sha256:f389e2997de33d038c5065fd85bff351fbdc62fa6d6371c7b947fc3bce8d437d
-    # via -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt
-iniconfig==2.1.0 \
-    --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \
-    --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760
-    # via
-    #   -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt
-    #   pytest
+basedpyright==1.35.0 \
+    --hash=sha256:2a7e0bd476623d48499e2b18ff6ed19dc28c51909cf9e1152ad355b5809049ad \
+    --hash=sha256:4f4f84023df5a0cd4ee154916ba698596682ac98bacfa22c941ed6aaf07bba4e
+iniconfig==2.3.0 \
+    --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \
+    --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12
 jinja2==3.1.6 \
     --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \
     --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
-    # via pytest-html
 markupsafe==3.0.2 \
     --hash=sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4 \
     --hash=sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30 \
@@ -80,54 +75,37 @@ markupsafe==3.0.2 \
     --hash=sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79 \
     --hash=sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430 \
     --hash=sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50
-    # via jinja2
-nodejs-wheel-binaries==22.16.0 \
-    --hash=sha256:2728972d336d436d39ee45988978d8b5d963509e06f063e80fe41b203ee80b28 \
-    --hash=sha256:2fffb4bf1066fb5f660da20819d754f1b424bca1b234ba0f4fa901c52e3975fb \
-    --hash=sha256:447ad796850eb52ca20356ad39b2d296ed8fef3f214921f84a1ccdad49f2eba1 \
-    --hash=sha256:4ae3cf22138891cb44c3ee952862a257ce082b098b29024d7175684a9a77b0c0 \
-    --hash=sha256:71f2de4dc0b64ae43e146897ce811f80ac4f9acfbae6ccf814226282bf4ef174 \
-    --hash=sha256:7f526ca6a132b0caf633566a2a78c6985fe92857e7bfdb37380f76205a10b808 \
-    --hash=sha256:986b715a96ed703f8ce0c15712f76fc42895cf09067d72b6ef29e8b334eccf64 \
-    --hash=sha256:d695832f026df3a0cf9a089d222225939de9d1b67f8f0a353b79f015aabbe7e2 \
-    --hash=sha256:dbfccbcd558d2f142ccf66d8c3a098022bf4436db9525b5b8d32169ce185d99e
-    # via
-    #   -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt
-    #   basedpyright
+nodejs-wheel-binaries==24.11.1 \
+    --hash=sha256:0e14874c3579def458245cdbc3239e37610702b0aa0975c1dc55e2cb80e42102 \
+    --hash=sha256:10197b1c9c04d79403501766f76508b0dac101ab34371ef8a46fcf51773497d0 \
+    --hash=sha256:376b9ea1c4bc1207878975dfeb604f7aa5668c260c6154dcd2af9d42f7734116 \
+    --hash=sha256:413dfffeadfb91edb4d8256545dea797c237bba9b3faefea973cde92d96bb922 \
+    --hash=sha256:5ef598101b0fb1c2bf643abb76dfbf6f76f1686198ed17ae46009049ee83c546 \
+    --hash=sha256:78bc5bb889313b565df8969bb7423849a9c7fc218bf735ff0ce176b56b3e96f0 \
+    --hash=sha256:c2741525c9874b69b3e5a6d6c9179a6fe484ea0c3d5e7b7c01121c8e5d78b7e2 \
+    --hash=sha256:c79a7e43869ccecab1cae8183778249cceb14ca2de67b5650b223385682c6239 \
+    --hash=sha256:cde41d5e4705266688a8d8071debf4f8a6fcea264c61292782672ee75a6905f9
 packaging==25.0 \
     --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \
     --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f
-    # via
-    #   -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt
-    #   pytest
 pluggy==1.6.0 \
     --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \
     --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746
-    # via
-    #   -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt
-    #   pytest
-pytest==8.3.5 \
-    --hash=sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820 \
-    --hash=sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845
-    # via
-    #   -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt
-    #   pytest-html
-    #   pytest-metadata
-    #   pytest-repeat
-    #   testing-utils
+pygments==2.19.2 \
+    --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \
+    --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b
+pytest==9.0.1 \
+    --hash=sha256:3e9c069ea73583e255c3b21cf46b8d3c56f6e3a1a8f6da94ccb0fcf57b9d73c8 \
+    --hash=sha256:67be0030d194df2dfa7b556f2e56fb3c3315bd5c8822c6951162b92b32ce7dad
 pytest-html==4.1.1 \
     --hash=sha256:70a01e8ae5800f4a074b56a4cb1025c8f4f9b038bba5fe31e3c98eb996686f07 \
     --hash=sha256:c8152cea03bd4e9bee6d525573b67bbc6622967b72b9628dda0ea3e2a0b5dd71
-    # via testing-utils
 pytest-metadata==3.1.1 \
     --hash=sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b \
     --hash=sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8
-    # via pytest-html
 pytest-repeat==0.9.4 \
     --hash=sha256:c1738b4e412a6f3b3b9e0b8b29fcd7a423e50f87381ad9307ef6f5a8601139f3 \
     --hash=sha256:d92ac14dfaa6ffcfe6917e5d16f0c9bc82380c135b03c2a5f412d2637f224485
-    # via testing-utils
 # WARNING: pip install will require the following package to be hashed.
 # Consider using a hashable URL like https://github.com/jazzband/pip-tools/archive/SOMECOMMIT.zip
 testing-utils @ git+https://github.com/eclipse-score/testing_tools.git@v0.3.0
-    # via -r feature_integration_tests/python_test_cases/requirements.txt

scripts/tooling/BUILD

@@ -15,17 +15,21 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 load("@score_tooling//python_basics:defs.bzl", "score_py_pytest")
 
-# In order to update the requirements, change the `requirements.in` file and run:
-# `bazel run //src:requirements.update`.
-# This will update the `requirements.txt` file.
-# To upgrade all dependencies to their latest versions, run:
-# `bazel run //src:requirements.update -- --upgrade`.
+# In order to update the requirements, change the `requirements.in` file and run
+# `bazel run //scripts/tooling:requirements.update`
+# This will update the `requirements.txt` file
+# To upgrade all dependencies to their latest versions, run
+# `bazel run //scripts/tooling:requirements.update -- --upgrade`
+
 compile_pip_requirements(
     name = "requirements",
     srcs = [
         "requirements.in",
         "@score_tooling//python_basics:requirements.txt",
     ],
+    extra_args = [
+        "--no-annotate",
+    ],
     requirements_txt = "requirements.txt",
     tags = [
         "manual",
@@ -58,6 +62,25 @@ py_binary(
     deps = [":cli"],
 )
 
+# Workflow scripts as executables
+py_binary(
+    name = "checkout_repos",
+    srcs = ["cli/workflow/checkout_repos.py"],
+    main = "cli/workflow/checkout_repos.py",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cli",
+        ":lib",
+    ] + all_requirements,
+)
+
+py_binary(
+    name = "recategorize_guidelines",
+    srcs = ["cli/workflow/recategorize_guidelines.py"],
+    main = "cli/workflow/recategorize_guidelines.py",
+    visibility = ["//visibility:public"],
+)
+
 # Tests target
 score_py_pytest(
     name = "tooling_tests",

scripts/tooling/cli/workflow/__init__.py

@@ -0,0 +1,12 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************

scripts/tooling/cli/workflow/checkout_repos.py

@@ -0,0 +1,109 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+"""
+Checkout all pinned repositories for CodeQL analysis.
+
+Clones repositories directly from known_good.json using GitPython library
+with GitHub token authentication to avoid rate limits.
+"""
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+from scripts.tooling.lib.git_operations import shallow_clone_repository
+from scripts.tooling.lib.known_good import load_known_good
+
+_LOG = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+
+def checkout_repo(name: str, url: str, ref: str, path: Path) -> None:
+    """
+    Checkout a single repository using git_operations library.
+
+    Args:
+        name: Repository name
+        url: Repository URL
+        ref: Git reference (branch, tag, or commit hash)
+        path: Local path to checkout into
+
+    Raises:
+        Exception: If checkout fails
+    """
+    _LOG.info(f"Checking out {name} ({ref}) to {path}")
+
+    shallow_clone_repository(url=url, path=path, ref=ref)
+
+
+def main() -> int:
+    """Main entry point for standalone execution."""
+    # When running with bazel, use BUILD_WORKING_DIRECTORY to find workspace root
+    workspace_root = Path(os.environ.get("BUILD_WORKING_DIRECTORY", "."))
+    known_good_path = workspace_root / "known_good.json"
+
+    if not known_good_path.exists():
+        _LOG.error(f"known_good.json not found at {known_good_path}")
+        return 1
+
+    try:
+        known_good = load_known_good(known_good_path)
+    except Exception as e:
+        _LOG.error(f"Failed to parse known_good.json: {e}")
+        return 1
+
+    # Extract target_sw modules
+    modules = known_good.modules.get("target_sw", {})
+    repo_count = len(modules)
+
+    _LOG.info(f"Checking out {repo_count} repositories from known_good.json...")
+
+    # Track successfully checked out repositories
+    repo_paths = []
+
+    # Checkout each repository
+    for name, module in modules.items():
+        url = module.repo
+        ref = module.version or module.branch or module.hash
+        # Use workspace-relative path
+        path = workspace_root / "repos" / name
+
+        try:
+            checkout_repo(name, url, ref, path)
+            repo_paths.append(str(path))
+        except Exception as e:
+            _LOG.error(f"Failed to checkout {name}: {e}")
+            return 1
+
+    # Output all paths (comma-separated for GitHub Actions compatibility)
+    repo_paths_output = ",".join(repo_paths)
+
+    # Write to GITHUB_OUTPUT if available
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        try:
+            with open(github_output, "a") as f:
+                f.write(f"repo_paths={repo_paths_output}\n")
+        except IOError as e:
+            _LOG.warning(f"Failed to write GITHUB_OUTPUT: {e}")
+
+    # Log summary
+    _LOG.info(f"Successfully checked out {len(repo_paths)} of {repo_count} repositories")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())