Fix build wheels workflow to allow triggering it on workflow dispatch (DataDog#21985)

* Fix build wheels workflow to allow running on dispatch

* Ensure image digest is persisted if we are going to build and lock files

* Unindent EOF

* Pin krb5 to avoid the failed build

* Revert "Pin krb5 to avoid the failed build"

This reverts commit 71154ef.

* Try new krb5 pin

* wip: Work in Progress

* Add back dependency on dcb alongside requests-kerberos

* Remove pin and opened a separate PR for that

* Remove redundant if condition and chomping pipes

* Extract complex logic to separate bash scripts

* Ensure we checkout the repo to get the scritps

Author: AAraKKe

.github/workflows/resolve-build-deps.yaml

@@ -1,6 +1,7 @@
 name: Resolve Dependencies and Build Wheels
 
 on:
+  workflow_dispatch:
   pull_request:
     branches:
     - master
@@ -26,39 +27,36 @@ env:
   # https://reproducible-builds.org/specs/source-date-epoch/
   SOURCE_DATE_EPOCH: "1580601600"
 
-jobs: 
+jobs:
   # measure-disk-usage.yml depends on this workflow being triggered and completed,
   # so it can wait for the build to calculate dependency sizes.
   # The 'on' setting ensures it runs, but this job cancels it if no dependency changes are detected.
 
-  check-dependency-changes:
-    name: Check dependency changes
+  check-should-run:
+    name: Check if build should run
     runs-on: ubuntu-22.04
     permissions:
       actions: write
+      contents: read
     outputs:
-      dependency_changed: ${{ steps.dependency-check.outputs.dependency_changed }}
       builder_changed: ${{ steps.dependency-check.outputs.builder_changed }}
+      should_run_build: ${{ steps.dependency-check.outputs.should_run_build }}
     steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
       - name: Define diff commits
         id: set_sha
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            PREV_SHA=${{ github.event.pull_request.base.sha }}
-            CURR_SHA=${{ github.event.pull_request.head.sha }}
-          else
-            PREV_SHA=${{ github.event.before }}
-            CURR_SHA=${{ github.sha }}
-          fi
-
-          echo "prev_sha=$PREV_SHA" >> $GITHUB_OUTPUT
-          echo "curr_sha=$CURR_SHA" >> $GITHUB_OUTPUT
-
-          echo "Current SHA: $CURR_SHA"
-          echo "Previous SHA: $PREV_SHA"
+        if: github.event_name != 'workflow_dispatch'
+        run: .github/workflows/scripts/resolve_deps_define_diff_commits.sh
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          EVENT_BEFORE: ${{ github.event.before }}
 
       - name: Get changed files
         id: changed-files
+        if: github.event_name != 'workflow_dispatch'
         run: |
           REPO="${{ github.repository }}"
 
@@ -70,50 +68,24 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Check for dependency changes
+      - name: Check if build should run
         id: dependency-check
-        run: |
-          FILES_CHANGED="${{ steps.changed-files.outputs.files_changed }}"
-
-          cat << EOF > dependency_files.txt
-          agent_requirements\.in
-          \.github/workflows/resolve-build-deps\.yaml
-          \.builders/
-          EOF
-
-          cat <<EOF > builder_files.txt
-          \.builders/
-          EOF
-
-          DEPENDENCY_CHANGED=$(
-              echo "$FILES_CHANGED" | \
-              grep -qf dependency_files.txt \
-              && echo "true" || echo "false"
-          ) 
-
-          BUILDER_CHANGED=$(
-              echo "$FILES_CHANGED" | \
-              grep -qf builder_files.txt \
-              && echo "true" || echo "false"
-          )
-
-
-          echo "dependency_changed=$DEPENDENCY_CHANGED" | tee -a $GITHUB_OUTPUT
-          echo "builder_changed=$BUILDER_CHANGED" | tee -a $GITHUB_OUTPUT
-
+        run: .github/workflows/scripts/resolve_deps_check_should_run.sh
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FILES_CHANGED: ${{ steps.changed-files.outputs.files_changed }}
 
   test:
     name: Run tests
     needs:
-      - check-dependency-changes
-    if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
+      - check-should-run
+    if: needs.check-should-run.outputs.should_run_build == 'true'
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Set up Python ${{ env.PYTHON_VERSION }} 
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
@@ -129,8 +101,8 @@ jobs:
   build:
     name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }}
     needs:
-      - check-dependency-changes
-    if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
+      - check-should-run
+    if: needs.check-should-run.outputs.should_run_build == 'true'
     runs-on: ${{ matrix.job.os }}
     strategy:
       fail-fast: false
@@ -161,8 +133,7 @@ jobs:
         python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Install management dependencies
-      run: |
-        pip install -r .builders/deps/host_dependencies.txt
+      run: pip install -r .builders/deps/host_dependencies.txt
 
     - name: Log in to GitHub Packages
       uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
@@ -172,29 +143,28 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Build image and wheels
-      if: needs.check-dependency-changes.outputs.builder_changed == 'true'
-      run: |-
-        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
+      if: needs.check-should-run.outputs.builder_changed == 'true'
+      run: python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3
 
     - name: Pull image and build wheels
-      if: needs.check-dependency-changes.outputs.builder_changed == 'false'
-      run: |-
+      if: needs.check-should-run.outputs.builder_changed == 'false'
+      run: |
         digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
         python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest
 
     - name: Publish image
-      if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true'
+      if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
       run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}
 
     - name: Save new image digest
-      if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true'
+      if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
       run: >-
         ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
         | cut -d '@' -f 2
         > ${{ env.OUT_DIR }}/image_digest
 
     - name: Persist current image digest
-      if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'false'
+      if: needs.check-should-run.outputs.builder_changed == 'false'
       run: >-
         jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json
         > ${{ env.OUT_DIR }}/image_digest
@@ -208,8 +178,8 @@ jobs:
   build-macos:
     name: Target macOS/${{ matrix.job.arch }} on ${{ matrix.job.os }}
     needs:
-      - check-dependency-changes
-    if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
+      - check-should-run
+    if: needs.check-should-run.outputs.should_run_build == 'true'
     runs-on: ${{ matrix.job.os }}
     strategy:
       fail-fast: false
@@ -229,7 +199,7 @@ jobs:
 
     steps:
     - name: Set up environment
-      run: |-
+      run: |
         # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
         # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
         brew remove --force --ignore-dependencies $(brew list --formula)
@@ -239,7 +209,7 @@ jobs:
       env:
         # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
         PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.9/python-3.13.9-macos11.pkg"
-      run: |-
+      run: |
         curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
         sudo installer -pkg python3.pkg -target /
 
@@ -255,16 +225,15 @@ jobs:
       id: cache-builder-root
       uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
       with:
-        path: |
-          ~/builder_root
+        path: ~/builder_root
         key: macos-${{ matrix.job.arch }}-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/resolve-build-deps.yml') }}
 
     - name: Run the build
       env:
         # This sets the minimum macOS version compatible for all built artifacts
         MACOSX_DEPLOYMENT_TARGET: "11.0" # https://docs.datadoghq.com/agent/supported_platforms/?tab=macos
         CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
-      run: |-
+      run: |
         # If we hit the cache, we can skip the builder setup
         if [[ ${CACHE_HIT} == "true" ]]; then
           ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup
@@ -282,11 +251,11 @@ jobs:
 
   publish:
     name: Publish artifacts and update lockfiles via PR
-    if: needs.check-dependency-changes.outputs.dependency_changed == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))) 
+    if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))))
     needs:
     - build
     - build-macos
-    - check-dependency-changes
+    - check-should-run
     runs-on: ubuntu-latest
 
     permissions:
@@ -328,7 +297,7 @@ jobs:
       run: python .builders/lock.py targets
 
     - name: Clean up repository
-      run: |-
+      run: |
         rm ${{ steps.auth.outputs.credentials_file_path }}
         rm -rf targets
 

.github/workflows/scripts/resolve_deps_check_should_run.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+if [ "$GITHUB_EVENT_NAME" == 'workflow_dispatch' ]; then
+  builder_changed="false"
+  should_run_build="true"
+else
+
+  cat << EOF > dependency_files.txt
+agent_requirements\.in
+\.github/workflows/resolve-build-deps\.yaml
+\.builders/
+EOF
+
+  cat <<EOF > builder_files.txt
+\.builders/
+EOF
+
+  should_run_build=$(
+      echo "$FILES_CHANGED" | \
+      grep -qf dependency_files.txt \
+      && echo "true" || echo "false"
+  )
+
+  builder_changed=$(
+      echo "$FILES_CHANGED" | \
+      grep -qf builder_files.txt \
+      && echo "true" || echo "false"
+  )
+fi
+
+echo "should_run_build=$should_run_build" | tee -a $GITHUB_OUTPUT
+echo "builder_changed=$builder_changed" | tee -a $GITHUB_OUTPUT

.github/workflows/scripts/resolve_deps_define_diff_commits.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
+  prev_sha=$PR_BASE_SHA
+  curr_sha=$PR_HEAD_SHA
+else
+  prev_sha=$EVENT_BEFORE
+  curr_sha=$GITHUB_SHA
+fi
+
+echo "prev_sha=$prev_sha" >> $GITHUB_OUTPUT
+echo "curr_sha=$curr_sha" >> $GITHUB_OUTPUT
+
+echo "Current SHA: $curr_sha"
+echo "Previous SHA: $prev_sha"