Replace paths: filter with content-hash gate in resolve-build-deps.yaml (DataDog#23458)

* Replace paths: filter with content-hash gate in resolve-build-deps.yaml

The workflow was over-triggering on merge commits whose push diffs swept in
already-landed .builders/ changes unrelated to the PR. Replace the paths:
filter with a content-hash gate backed by .builders/inputs_hash.py.

- inputs_hash.py: replace COMMON_INPUTS (literal paths) with SHARED_INPUTS
  and RESOLUTION_INPUTS (glob patterns); add compute_target, pinned_target,
  compute_resolution, pinned_resolution, verify_resolution, status; replace
  old compute/pinned CLI subcommands with status and verify-resolution.
- upload.py: import inputs_hash, extend _write_builder_inputs to accept a
  resolution hash, emit [resolution] + [images] TOML sections, pass
  compute_resolution() at the call site.
- resolve-build-deps.yaml: drop paths: filter; add gate job that runs
  inputs_hash.py status; add if: needs_resolution == 'true' to test job;
  read rebuild_<target> flags from gate outputs in build job.
- verify-deps-pin.yaml: new workflow that runs inputs_hash.py verify-resolution
  on gh-readonly-queue/** pushes to block merges with a stale resolution pin.
- tests/test_inputs_hash.py: new test file covering _hash_paths stability,
  compute/pinned/verify for targets and resolution, status fresh/stale cases,
  CLI output shape, and a coverage test that fails CI if a new .builders/ file
  is not classified.

NOTE: verify-deps-pin must be added to required status checks on master and
release branches before this fully closes the concurrent-PR race window.

* review feedback

* Update dependency resolution

* tests and introduce gate runs for master and release branches.

* Update dependency resolution

* implement clear feedback

* Overhaul hashing in response to reviews; stop writing metadata.json

* fix type hint

* Update dependency resolution

* Update dependency resolution

* more feedback

* Update dependency resolution

---------

Co-authored-by: dd-agent-integrations-bot[bot] <dd-agent-integrations-bot[bot]@users.noreply.github.com>

Author: iliakur

.builders/inputs_hash.py

@@ -0,0 +1,7 @@
+[
+  {"platform": "linux",   "arch": "x86_64",  "runner_os": "ubuntu-22.04"},
+  {"platform": "linux",   "arch": "aarch64", "runner_os": "ubuntu-22.04-arm"},
+  {"platform": "windows", "arch": "x86_64",  "runner_os": "windows-2022"},
+  {"platform": "macos",   "arch": "x86_64",  "runner_os": "macos-14-large"},
+  {"platform": "macos",   "arch": "aarch64", "runner_os": "macos-14"}
+]

.builders/targets.json

@@ -4,6 +4,8 @@
 from zipfile import ZipFile
 
 import pytest
+
+import inputs_hash
 import upload
 
 
@@ -466,7 +468,7 @@ def test_lockfile_generation(tmp_path, setup_targets_dir, frozen_timestamp):
     with mock.patch.object(upload, "RESOLUTION_DIR", fake_deps_dir), \
          mock.patch.object(upload, "LOCK_FILE_DIR", fake_resolved_dir):
 
-        upload.generate_lockfiles(targets_dir, lockfile)
+        upload.generate_lockfiles(targets_dir, lockfile, 'dummy_hash')
         lock_files = list(fake_resolved_dir.glob("*.txt"))
         assert lock_files, "No lock files generated"
         lockfile_map = {lock_file.name: lock_file.read_text().strip() for lock_file in lock_files}
@@ -496,7 +498,7 @@ def test_generate_lockfiles_skips_targets_with_no_wheels(tmp_path):
 
     with mock.patch.object(upload, "RESOLUTION_DIR", fake_deps_dir), \
          mock.patch.object(upload, "LOCK_FILE_DIR", fake_resolved_dir):
-        upload.generate_lockfiles(str(tmp_path / "targets"), lockfile)
+        upload.generate_lockfiles(str(tmp_path / "targets"), lockfile, 'dummy_hash')
 
     lock_files = {f.name for f in fake_resolved_dir.glob("*.txt")}
     assert lock_files == {f"linux-x86_64_{upload.CURRENT_PYTHON_VERSION}.txt"}
@@ -516,7 +518,48 @@ def test_generate_lockfiles_accepts_string_path(tmp_path):
     with mock.patch.object(upload, "RESOLUTION_DIR", fake_deps_dir), \
          mock.patch.object(upload, "LOCK_FILE_DIR", fake_resolved_dir):
         # Should not raise TypeError: unsupported operand type(s) for /: 'str' and 'str'
-        upload.generate_lockfiles(str(tmp_path / "targets"), lockfile)
+        upload.generate_lockfiles(str(tmp_path / "targets"), lockfile, 'dummy_hash')
+
+
+def test_generate_lockfiles_writes_resolution_pin_and_target_inputs(tmp_path):
+    """generate_lockfiles writes the gate-passed resolution_hash plus per-target inputs_sha256 into builder_inputs.toml.
+
+    Pins the four-hop contract (gate output → workflow env → CLI flag →
+    write call) so a refactor that drops resolution_hash anywhere along the
+    way fails here rather than producing a stale or empty [resolution]
+    section in the bot commit.
+    """
+    import tomllib
+
+    lockfile = {
+        'linux-x86_64': ['dep @ https://example.com/dep.whl#sha256=abc', ''],
+        'macos-x86_64': ['dep @ https://example.com/dep-macos.whl#sha256=def', ''],
+    }
+
+    fake_deps_dir = tmp_path / '.deps'
+    fake_resolved_dir = fake_deps_dir / 'resolved'
+    fake_deps_dir.mkdir()
+    fake_resolved_dir.mkdir()
+    targets_dir = tmp_path / 'targets'
+    (targets_dir / 'linux-x86_64').mkdir(parents=True)
+    (targets_dir / 'linux-x86_64' / 'inputs_sha256').write_text('linux-hash', encoding='utf-8')
+    (targets_dir / 'linux-x86_64' / 'image_digest').write_text('sha256:linux-digest', encoding='utf-8')
+    (targets_dir / 'macos-x86_64').mkdir(parents=True)
+    (targets_dir / 'macos-x86_64' / 'inputs_sha256').write_text('macos-hash', encoding='utf-8')
+
+    with mock.patch.object(upload, 'RESOLUTION_DIR', fake_deps_dir), \
+         mock.patch.object(upload, 'LOCK_FILE_DIR', fake_resolved_dir):
+        upload.generate_lockfiles(targets_dir, lockfile, 'gate-passed-hash')
+
+    pin_path = fake_deps_dir / 'builder_inputs.toml'
+    with pin_path.open('rb') as f:
+        pin = tomllib.load(f)
+
+    assert pin[inputs_hash.SECTION_RESOLUTION][inputs_hash.HASH_KEY] == 'gate-passed-hash'
+    assert pin[inputs_hash.SECTION_IMAGES] == {
+        'linux-x86_64': 'linux-hash',
+        'macos-x86_64': 'macos-hash',
+    }
 
 
 def test_collect_and_validate_wheels(tmp_path):

.builders/tests/test_inputs_hash.py

@@ -12,6 +12,8 @@
 
 from google.cloud import storage
 
+import inputs_hash
+
 if TYPE_CHECKING:
     from google.cloud.storage.bucket import Bucket as GCSBucket
 
@@ -20,7 +22,6 @@
 REPO_DIR = BUILDER_DIR.parent
 RESOLUTION_DIR = REPO_DIR / '.deps'
 LOCK_FILE_DIR = RESOLUTION_DIR / 'resolved'
-DIRECT_DEP_FILE = REPO_DIR / 'agent_requirements.in'
 CACHE_CONTROL = 'public, max-age=15'
 VALID_PROJECT_NAME = re.compile(r'^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', re.IGNORECASE)
 UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+')
@@ -274,18 +275,9 @@ def upload_string(self, content: str, blob_path: str, content_type: str = 'text/
             blob.patch()
 
 
-def generate_lockfiles(targets_dir, lockfiles):
+def generate_lockfiles(targets_dir, lockfiles, resolution_hash):
     targets_dir = Path(targets_dir)
     LOCK_FILE_DIR.mkdir(parents=True, exist_ok=True)
-    with RESOLUTION_DIR.joinpath('metadata.json').open('w', encoding='utf-8') as f:
-        contents = json.dumps(
-            {
-                'sha256': sha256(DIRECT_DEP_FILE.read_bytes()).hexdigest(),
-            },
-            indent=2,
-            sort_keys=True,
-        )
-        f.write(f'{contents}\n')
 
     image_digests = {}
     builder_inputs = {}
@@ -311,31 +303,10 @@ def generate_lockfiles(targets_dir, lockfiles):
         contents = json.dumps(image_digests, indent=2, sort_keys=True)
         f.write(f'{contents}\n')
 
-    _write_builder_inputs(RESOLUTION_DIR / 'builder_inputs.toml', builder_inputs)
-
-
-_BUILDER_INPUTS_HEADER = """\
-# Content hashes of the inputs that determine each builder image.
-#
-# The `Resolve Dependencies and Build Wheels` workflow compares these
-# hashes against hashes computed from the working tree (via
-# .builders/inputs_hash.py) to decide whether to rebuild a builder image
-# from scratch or pull the existing one by digest from image_digests.json.
-#
-# This file is rewritten by .builders/upload.py whenever dependency
-# resolution publishes new artifacts and should not be edited by hand.
-# The set of files covered by each hash is defined by COMMON_INPUTS in
-# .builders/inputs_hash.py plus everything under .builders/images/<target>/.
-
-[inputs]
-"""
-
-
-def _write_builder_inputs(path: Path, hashes: dict[str, str]) -> None:
-    lines = [_BUILDER_INPUTS_HEADER.rstrip('\n')]
-    for target in sorted(hashes):
-        lines.append(f'{target} = "{hashes[target]}"')
-    path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
+    inputs_hash.write_pinned_hashes(
+        RESOLUTION_DIR / 'builder_inputs.toml',
+        inputs_hash.PinnedHashes(resolution=resolution_hash, images=builder_inputs),
+    )
 
 
 def upload(targets_dir: Path, bucket: Bucket | None = None) -> dict[str, list[str]]:
@@ -384,6 +355,14 @@ def upload(targets_dir: Path, bucket: Bucket | None = None) -> dict[str, list[st
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
     parser.add_argument('targets_dir')
+    parser.add_argument(
+        '--resolution-hash',
+        required=True,
+        help=(
+            'Hex sha256 of the resolution inputs, computed by the gate job '
+            'and passed in to avoid recomputing against the working tree.'
+        ),
+    )
     args = parser.parse_args()
     lockfiles = upload(args.targets_dir)
-    generate_lockfiles(args.targets_dir, lockfiles)
+    generate_lockfiles(args.targets_dir, lockfiles, args.resolution_hash)

.builders/tests/test_upload.py

@@ -1,16 +1,21 @@
-# Content hashes of the inputs that determine each builder image.
+# Content hashes of the inputs that determine the resolution pipeline and each
+# builder image.
 #
-# The `Resolve Dependencies and Build Wheels` workflow compares these
-# hashes against hashes computed from the working tree (via
-# .builders/inputs_hash.py) to decide whether to rebuild a builder image
-# from scratch or pull the existing one by digest from image_digests.json.
+# The `Resolve Dependencies and Build Wheels` workflow uses these hashes to
+# gate the full pipeline (resolution.hash) and to decide whether to rebuild a
+# builder image from scratch or pull the existing one by digest (images.*).
+# The `verify-deps-pin` workflow checks resolution.hash on merge-queue refs.
 #
-# This file is rewritten by .builders/upload.py whenever dependency
-# resolution publishes new artifacts and should not be edited by hand.
-# The set of files covered by each hash is defined by COMMON_INPUTS in
-# .builders/inputs_hash.py plus everything under .builders/images/<target>/.
+# This file is rewritten by .builders/upload.py whenever dependency resolution
+# publishes new artifacts and should not be edited by hand.
+# Hash inputs are defined in .builders/inputs_hash.py (SHARED_INPUTS,
+# RESOLUTION_INPUTS).
+[resolution]
+hash = "090005fe2acdb03816c94f40949836db31b29350bc0746a0a7bc8c5d896119c8"
 
-[inputs]
-linux-aarch64 = "e3b9a63fa95c9d58a54926fd42d1a20ae738a6c05ff8338d995b8cd9c5497ef2"
-linux-x86_64 = "58a8e70bf5f870e210d586662843cdc9948b5d90638e3487f1e07a2b10f27f19"
-windows-x86_64 = "130572854fdcd9ca5d021af13087afa7384b1e659959087db21c77ac144ad313"
+[images]
+linux-aarch64 = "5239191575da3521adfbb9992114201c71c65ff3056de4ae2e475390d218d194"
+linux-x86_64 = "83d13d3171972f56b1336d325eecae73bf5e2329a24142f25e68ae50c542ce6a"
+macos-aarch64 = "8a4b86487b09da7dda2165487464a2e80bbe623a6db56b5eb23db35eec3ea4f0"
+macos-x86_64 = "0c4f87d5e5df769f19ad8527208e8cdcc7268b1741e1b5e67483133c9c0073c1"
+windows-x86_64 = "d7cc3720c0ac20a28c20d0fd606d6d0a84023e97682c39994ac99c1cdef9e817"

.builders/upload.py

@@ -1,5 +1,5 @@
 {
-  "linux-aarch64": "sha256:bced710c0b1812c224caaa951b412ba3fbf2d299dcb4cb001b58898cd9e119ae",
-  "linux-x86_64": "sha256:82a4605484daf57ff3edfc2c4a10d19cb3ee592ba5cdf6a53605c99a76856ae3",
-  "windows-x86_64": "sha256:7ebc2f51eca870f551f663159b399e2cd98a15830f2a495a775dfd124f7f859b"
+  "linux-aarch64": "sha256:293e22087f45e4f460a527f2583c9592ae702a55b318cc6066f255e60470a40e",
+  "linux-x86_64": "sha256:57380fd04afcf90a50087ad3d3e32c2075a801bab93478be4a83c0a2cabeb867",
+  "windows-x86_64": "sha256:5f99e8d346605ce93a0656908bbd4da6258ce9b67798c18a9a327a5f08403be7"
 }