Add webhook health precheck to cluster sanity checks

.flake8

@@ -12,6 +12,7 @@ exclude =
     ansible/*,
     docs/*,
     .cache/*
+    utilities/unittests/*
 
 fcn_exclude_functions =
     re,

.github/labeler.yml

@@ -1,8 +1,43 @@
-# Add 'sig-network' label to any changes within 'tests/network' folder
-# or any subfolders
+# Adds a 'sig-<sig>' label when there are changes in specific tests/** subtrees
+# and/or their corresponding utilities/*.py files.
+
+sig-chaos:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'tests/chaos/**'
+
+sig-infra:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'tests/infrastructure/**'
+      - 'utilities/ssp.py'
+
+sig-iuo:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'tests/{install_upgrade_operators,observability}/**'
+      - 'utilities/{hco,monitoring,must_gather,operator}.py'
+
 sig-network:
   - changed-files:
     - any-glob-to-any-file:
       - 'tests/network/**'
       - 'libs/net/**'
       - 'utilities/network.py'
+
+sig-scale:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'tests/scale/**'
+
+sig-storage:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'tests/{data_protection,storage}/**'
+      - 'utilities/storage.py'
+
+sig-virt:
+  - changed-files:
+    - any-glob-to-any-file:
+      - 'tests/virt/**'
+      - 'utilities/virt.py'

.github/workflows/cherrypick-to-4.99.yml

@@ -0,0 +1,337 @@
+## Disabling this workflow for now until we decide how to handle the different branches
+
+#name: PR Cherry-pick to cnv-4.99
+#
+#on:
+#  push:
+#    branches:
+#      - main
+#
+#concurrency:
+#  group: cherry-pick-cnv-4.99
+#  cancel-in-progress: true
+#
+#jobs:
+#  cherry-pick-pr:
+#    runs-on: ubuntu-latest
+#    permissions:
+#      contents: write
+#      pull-requests: write
+#      issues: write
+#      actions: read
+#
+#    steps:
+#      - name: Checkout repository
+#        uses: actions/checkout@v4
+#        with:
+#          fetch-depth: 0
+#          token: ${{ secrets.GITHUB_TOKEN }}
+#
+#      - name: Check if push is from PR merge
+#        id: check-pr
+#        run: |
+#          set -euo pipefail
+#
+#          # Get the commit SHA that was just pushed
+#          COMMIT_SHA="${{ github.sha }}"
+#          echo "Checking commit: $COMMIT_SHA"
+#
+#          # Validate SHA format
+#          if [[ ! "$COMMIT_SHA" =~ ^[a-f0-9]{40}$ ]]; then
+#            echo "Invalid commit SHA format: $COMMIT_SHA"
+#            exit 1
+#          fi
+#
+#          # Use GitHub API to detect if commit is associated with a PR
+#          echo "Using GitHub API to check for associated PR..."
+#
+#          # Query the commits/{sha}/pulls API endpoint
+#          if ! PR_RESPONSE=$(curl -sf \
+#            --connect-timeout 5 \
+#            --max-time 30 \
+#            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+#            -H "Accept: application/vnd.github+json" \
+#            -H "X-GitHub-Api-Version: 2022-11-28" \
+#            "https://api.github.com/repos/${{ github.repository }}/commits/$COMMIT_SHA/pulls"); then
+#            echo "Failed to query GitHub API for commit $COMMIT_SHA"
+#            echo "is_pr=false" >> "$GITHUB_OUTPUT"
+#            exit 0
+#          fi
+#
+#          # Check if any PRs are associated with this commit
+#          PR_COUNT=$(echo "$PR_RESPONSE" | jq -r 'length')
+#
+#          if [ "$PR_COUNT" -eq 0 ]; then
+#            echo "No PR associated with commit $COMMIT_SHA - skipping"
+#            echo "is_pr=false" >> "$GITHUB_OUTPUT"
+#            exit 0
+#          fi
+#
+#          # Get the first (most relevant) PR from the response
+#          if ! PR_DATA=$(echo "$PR_RESPONSE" | jq -r '.[0]'); then
+#            echo "Failed to parse PR data from API response"
+#            echo "is_pr=false" >> "$GITHUB_OUTPUT"
+#            exit 0
+#          fi
+#
+#          # Extract PR details from API response with safe fallbacks
+#          PR_NUMBER=$(echo "$PR_DATA" | jq -r '.number // ""')
+#          PR_OWNER=$(echo "$PR_DATA" | jq -r '.head.repo.owner.login // .base.repo.owner.login // ""')
+#          PR_REPO=$(echo "$PR_DATA" | jq -r '.head.repo.name // .base.repo.name // ""')
+#          BRANCH_NAME=$(echo "$PR_DATA" | jq -r '.head.ref // "deleted-branch"')
+#
+#          # Use repository defaults when head repo is unavailable (fork deleted)
+#          if [ -z "$PR_OWNER" ] || [ "$PR_OWNER" = "null" ]; then
+#            PR_OWNER="${{ github.repository_owner }}"
+#          fi
+#          if [ -z "$PR_REPO" ] || [ "$PR_REPO" = "null" ]; then
+#            PR_REPO="${{ github.event.repository.name }}"
+#          fi
+#          if [ -z "$BRANCH_NAME" ] || [ "$BRANCH_NAME" = "null" ]; then
+#            BRANCH_NAME="deleted-branch"
+#          fi
+#
+#          if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then
+#            echo "Invalid PR number from API response"
+#            echo "is_pr=false" >> "$GITHUB_OUTPUT"
+#            exit 0
+#          fi
+#
+#          echo "Found PR associated with commit (with safe fallbacks applied):"
+#          echo "PR Number: $PR_NUMBER"
+#          echo "PR Owner: $PR_OWNER"
+#          echo "PR Repo: $PR_REPO"
+#          echo "Branch: $BRANCH_NAME"
+#
+#          echo "is_pr=true" >> "$GITHUB_OUTPUT"
+#          echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
+#          echo "pr_owner=$PR_OWNER" >> "$GITHUB_OUTPUT"
+#          echo "pr_repo=$PR_REPO" >> "$GITHUB_OUTPUT"
+#          echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT"
+#          echo "commit_sha=$COMMIT_SHA" >> "$GITHUB_OUTPUT"
+#
+#      - name: Get PR details via GitHub API
+#        if: steps.check-pr.outputs.is_pr == 'true'
+#        id: get-pr-details
+#        run: |
+#          set -euo pipefail
+#
+#          PR_NUMBER="${{ steps.check-pr.outputs.pr_number }}"
+#
+#          # Get PR details from GitHub API with error handling
+#          if ! PR_DATA=$(curl -sf \
+#            --connect-timeout 5 \
+#            --max-time 30 \
+#            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+#            -H "Accept: application/vnd.github+json" \
+#            -H "X-GitHub-Api-Version: 2022-11-28" \
+#            "https://api.github.com/repos/${{ github.repository }}/pulls/$PR_NUMBER"); then
+#            echo "Failed to fetch PR data for PR #$PR_NUMBER"
+#            exit 1
+#          fi
+#
+#          # Extract relevant information with jq
+#          if ! PR_TITLE=$(echo "$PR_DATA" | jq -r '.title // ""'); then
+#            echo "Failed to extract PR title"
+#            exit 1
+#          fi
+#
+#          if ! PR_AUTHOR=$(echo "$PR_DATA" | jq -r '.user.login // ""'); then
+#            echo "Failed to extract PR author"
+#            exit 1
+#          fi
+#
+#          if ! PR_BODY=$(echo "$PR_DATA" | jq -r '.body // ""'); then
+#            echo "Failed to extract PR body"
+#            exit 1
+#          fi
+#
+#          # Use random delimiters to safely set multiline outputs
+#          DELIMITER_TITLE=$(openssl rand -hex 16)
+#          DELIMITER_AUTHOR=$(openssl rand -hex 16)
+#          DELIMITER_BODY=$(openssl rand -hex 16)
+#
+#          {
+#            echo "pr_title<<$DELIMITER_TITLE"
+#            echo "$PR_TITLE"
+#            echo "$DELIMITER_TITLE"
+#          } >> "$GITHUB_OUTPUT"
+#
+#          {
+#            echo "pr_author<<$DELIMITER_AUTHOR"
+#            echo "$PR_AUTHOR"
+#            echo "$DELIMITER_AUTHOR"
+#          } >> "$GITHUB_OUTPUT"
+#
+#          {
+#            echo "pr_body<<$DELIMITER_BODY"
+#            echo "$PR_BODY"
+#            echo "$DELIMITER_BODY"
+#          } >> "$GITHUB_OUTPUT"
+#
+#      - name: Setup Git configuration
+#        if: steps.check-pr.outputs.is_pr == 'true'
+#        run: |
+#          git config user.name "github-actions[bot]"
+#          git config user.email "github-actions[bot]@users.noreply.github.com"
+#
+#      - name: Prepare cherry-pick to cnv-4.99
+#        if: steps.check-pr.outputs.is_pr == 'true'
+#        id: cherry-pick
+#        run: |
+#          set -euo pipefail
+#
+#          TARGET_BRANCH="cnv-4.99"
+#          COMMIT_SHA="${{ steps.check-pr.outputs.commit_sha }}"
+#          PR_NUMBER="${{ steps.check-pr.outputs.pr_number }}"
+#
+#          echo "Preparing to cherry-pick $COMMIT_SHA to $TARGET_BRANCH"
+#
+#          # Fetch and checkout target branch
+#          if ! git fetch origin "$TARGET_BRANCH"; then
+#            echo "Failed to fetch target branch: $TARGET_BRANCH"
+#            exit 1
+#          fi
+#
+#          if ! git checkout -B "$TARGET_BRANCH" "origin/$TARGET_BRANCH"; then
+#            echo "Failed to checkout target branch: $TARGET_BRANCH"
+#            exit 1
+#          fi
+#
+#          # Check if commit was already cherry-picked via -x trailer
+#          if git log --grep="(cherry picked from commit $COMMIT_SHA)" --oneline HEAD | grep -q "(cherry picked from commit $COMMIT_SHA)"; then
+#            echo "Commit $COMMIT_SHA already cherry-picked to $TARGET_BRANCH (found -x trailer) - skipping"
+#            echo "skip_cherry_pick=true" >> "$GITHUB_OUTPUT"
+#            exit 0
+#          fi
+#
+#          # Check if commit already exists on target branch
+#          if git merge-base --is-ancestor "$COMMIT_SHA" HEAD; then
+#            echo "Commit $COMMIT_SHA already exists on $TARGET_BRANCH - skipping"
+#            echo "skip_cherry_pick=true" >> "$GITHUB_OUTPUT"
+#            exit 0
+#          fi
+#
+#          echo "skip_cherry_pick=false" >> "$GITHUB_OUTPUT"
+#
+#          # Attempt cherry-pick to working directory (let peter-evans handle the rest)
+#          echo "Attempting to cherry-pick $COMMIT_SHA"
+#
+#          # Check if this is a merge commit (has multiple parents)
+#          PARENT_COUNT=$(git rev-list --parents -n 1 "$COMMIT_SHA" | wc -w)
+#          PARENT_COUNT=$((PARENT_COUNT - 1))  # Subtract 1 because rev-list includes the commit itself
+#
+#          # Build cherry-pick command based on commit type
+#          if [ "$PARENT_COUNT" -gt 1 ]; then
+#            # Merge commit - use -m 1 to specify mainline parent
+#            CHERRY_PICK_CMD="git cherry-pick --no-commit -x -m 1 $COMMIT_SHA"
+#          else
+#            # Regular commit - no need for -m flag
+#            CHERRY_PICK_CMD="git cherry-pick --no-commit -x $COMMIT_SHA"
+#          fi
+#
+#          echo "Commit has $PARENT_COUNT parent(s), using command: $CHERRY_PICK_CMD"
+#
+#          if $CHERRY_PICK_CMD; then
+#            echo "Cherry-pick successful - changes ready for PR creation"
+#            echo "has_conflicts=false" >> "$GITHUB_OUTPUT"
+#          else
+#            echo "Cherry-pick failed with conflicts - staging for peter-evans"
+#
+#            # Get list of conflicted files
+#            CONFLICTED_FILES=$(git diff --name-only --diff-filter=U | tr '\n' ' ' || echo "Unable to determine conflicted files")
+#            echo "Conflicted files: $CONFLICTED_FILES"
+#
+#            # Use random delimiter for conflicted files output
+#            DELIMITER_CONFLICTS=$(openssl rand -hex 16)
+#            {
+#              echo "conflicted_files<<$DELIMITER_CONFLICTS"
+#              echo "$CONFLICTED_FILES"
+#              echo "$DELIMITER_CONFLICTS"
+#            } >> "$GITHUB_OUTPUT"
+#
+#            echo "has_conflicts=true" >> "$GITHUB_OUTPUT"
+#
+#            # Refresh index and stage all changes (including conflicts) for peter-evans to handle
+#            # This resolves the "you need to resolve your current index first" error
+#
+#            git add -A :/
+#
+#            git update-index --refresh -q || true
+#          fi
+#
+#      - name: Create PR for successful cherry-pick
+#        if: steps.check-pr.outputs.is_pr == 'true' && steps.cherry-pick.outputs.skip_cherry_pick == 'false' && steps.cherry-pick.outputs.has_conflicts == 'false'
+#        id: create-pr-success
+#        uses: peter-evans/create-pull-request@v7
+#        with:
+#          token: ${{ secrets.GITHUB_TOKEN }}
+#          branch: "cherry-pick-pr-${{ steps.check-pr.outputs.pr_number }}-${{ github.run_id }}"
+#          title: "Cherry-pick: ${{ steps.get-pr-details.outputs.pr_title }} (PR #${{ steps.check-pr.outputs.pr_number }})"
+#          body: |
+#            ## 🍒 Automated Cherry-pick from PR
+#
+#            This PR contains the cherry-pick of **PR #${{ steps.check-pr.outputs.pr_number }}** from `${{ steps.check-pr.outputs.pr_owner }}/${{ steps.check-pr.outputs.pr_repo }}`.
+#
+#            **Original PR:** ${{ steps.get-pr-details.outputs.pr_title }}
+#            **Author:** @${{ steps.get-pr-details.outputs.pr_author }}
+#            **Merge Commit:** ${{ steps.check-pr.outputs.commit_sha }}
+#
+#            The cherry-pick was applied cleanly to `cnv-4.99` without conflicts.
+#
+#            **Original PR Link:** ${{ github.server_url }}/${{ github.repository }}/pull/${{ steps.check-pr.outputs.pr_number }}
+#            **Original Commit:** ${{ github.server_url }}/${{ github.repository }}/commit/${{ steps.check-pr.outputs.commit_sha }}
+#
+#            ### Original PR Description
+#            ${{ steps.get-pr-details.outputs.pr_body }}
+#          base: "cnv-4.99"
+#          delete-branch: true
+#          labels: cherry-pick-to-4.99
+#
+#      - name: Check if PR was created or already exists
+#        if: steps.check-pr.outputs.is_pr == 'true' && steps.cherry-pick.outputs.skip_cherry_pick == 'false' && steps.cherry-pick.outputs.has_conflicts == 'false'
+#        run: |
+#          if [ "${{ steps.create-pr-success.outputs.pull-request-operation }}" = "none" ]; then
+#            echo "No changes were found to create a PR - changes might already exist on target branch"
+#          else
+#            echo "PR created successfully: ${{ steps.create-pr-success.outputs.pull-request-url }}"
+#          fi
+#
+#      - name: Create draft PR for conflicted cherry-pick
+#        if: steps.check-pr.outputs.is_pr == 'true' && steps.cherry-pick.outputs.skip_cherry_pick == 'false' && steps.cherry-pick.outputs.has_conflicts == 'true'
+#        uses: peter-evans/create-pull-request@v7
+#        with:
+#          token: ${{ secrets.GITHUB_TOKEN }}
+#          branch: "cherry-pick-pr-${{ steps.check-pr.outputs.pr_number }}-${{ github.run_id }}-conflicts"
+#          title: "🚨 Cherry-pick Conflicts: ${{ steps.get-pr-details.outputs.pr_title }} (PR #${{ steps.check-pr.outputs.pr_number }})"
+#          body: |
+#            ## ⚠️ Cherry-pick Conflicts - Manual Resolution Required
+#
+#            The automatic cherry-pick of **PR #${{ steps.check-pr.outputs.pr_number }}** from `${{ steps.check-pr.outputs.pr_owner }}/${{ steps.check-pr.outputs.pr_repo }}` resulted in conflicts.
+#
+#            **Original PR:** ${{ steps.get-pr-details.outputs.pr_title }}
+#            **Author:** @${{ steps.get-pr-details.outputs.pr_author }}
+#            **Merge Commit:** ${{ steps.check-pr.outputs.commit_sha }}
+#
+#            ### 🔧 How to resolve:
+#            1. Check out this branch locally: `git fetch origin && git checkout cherry-pick-pr-${{ steps.check-pr.outputs.pr_number }}-${{ github.run_id }}-conflicts`
+#            2. Resolve conflicts in the affected files
+#            3. Test your changes thoroughly
+#            4. Commit your resolution: `git add . && git commit`
+#            5. Push changes: `git push origin cherry-pick-pr-${{ steps.check-pr.outputs.pr_number }}-${{ github.run_id }}-conflicts`
+#            6. Mark this PR as ready for review
+#
+#            ### 📁 Files with conflicts:
+#            ```
+#            ${{ steps.cherry-pick.outputs.conflicted_files }}
+#            ```
+#
+#            **Original PR Link:** ${{ github.server_url }}/${{ github.repository }}/pull/${{ steps.check-pr.outputs.pr_number }}
+#            **Original Commit:** ${{ github.server_url }}/${{ github.repository }}/commit/${{ steps.check-pr.outputs.commit_sha }}
+#
+#            ### Original PR Description
+#            ${{ steps.get-pr-details.outputs.pr_body }}
+#          base: "cnv-4.99"
+#          delete-branch: true
+#          draft: true
+#          labels: cherry-pick-conflicts, cherry-pick-to-4.99