feat(downloader): resolve hash-prefixed targets via N.targets.json

The v2 TUF repository uses consistent-snapshot format: pointer files are
stored as {sha256}.{version}.json on S3. Two changes to support this:

1. _make_updater now sets UpdaterConfig(prefix_targets_with_hash=True) so
   the TUF Updater resolves hash-prefixed paths automatically when calling
   download_target().

2. get_pointer() now parses N.targets.json (after Updater.refresh()) to
   enumerate available versions for the project. This replaces the removed
   latest.json: when version=None, _resolve_version() scans all
   <project>/<ver>.json entries in targets metadata and returns the highest
   stable PEP 440 version.

The disable_verification path fetches the metadata chain (timestamp →
snapshot → targets) without verifying signatures to find the hash-prefixed
URL, then fetches the pointer directly.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: dkirov-dd

datadog_checks_downloader/datadog_checks/downloader/download_v2.py

@@ -8,6 +8,11 @@
 
     targets/<project>/<version>.json   (versioned pointer)
 
+Target files are content-addressed on S3 (consistent-snapshot format):
+the actual file at ``targets/<project>/<version>.json`` is stored as
+``targets/<project>/{sha256}.{version}.json``.  The TUF Updater resolves
+the hash-prefixed path automatically via ``N.targets.json`` metadata.
+
 Each pointer file contains:
 
     {
@@ -33,14 +38,17 @@
 import urllib.request
 from pathlib import Path
 
+from packaging.version import InvalidVersion, Version
+from tuf.api.metadata import Metadata, Snapshot, Targets, Timestamp
 from tuf.ngclient import Updater
+from tuf.ngclient.config import UpdaterConfig
 
 from .exceptions import DigestMismatch, TargetNotFoundError
 
-V2_REPOSITORY_URL = "https://agent-integration-wheels-prod.s3.amazonaws.com"
-
 logger = logging.getLogger(__name__)
 
+V2_REPOSITORY_URL = "https://agent-integration-wheels-prod.s3.amazonaws.com"
+
 
 class TUFPointerDownloader:
     """Downloads Datadog integration wheels from a v2 TUF repository."""
@@ -87,48 +95,110 @@ def _make_updater(self, metadata_dir: Path, target_dir: Path) -> Updater:
             metadata_base_url=f'{self._repository_url}/metadata/',
             target_base_url=f'{self._repository_url}/targets/',
             target_dir=str(target_dir),
+            config=UpdaterConfig(prefix_targets_with_hash=True),
         )
 
+    @staticmethod
+    def _resolve_version(project: str, version: str | None, targets: Targets) -> str:
+        """Return the concrete version to fetch.
+
+        If *version* is given, return it as-is (TUF will raise if absent).
+        If None, scan *targets* for all ``<project>/<ver>.json`` entries and
+        return the highest stable PEP 440 version.
+        """
+        if version is not None:
+            return version
+
+        prefix = f"{project}/"
+        stable: list[Version] = []
+        for target_path in targets.targets:
+            if not (target_path.startswith(prefix) and target_path.endswith(".json")):
+                continue
+            stem = target_path[len(prefix): -len(".json")]
+            try:
+                v = Version(stem)
+            except InvalidVersion:
+                continue
+            if not v.is_prerelease:
+                stable.append(v)
+
+        if not stable:
+            raise TargetNotFoundError(f"No stable releases found for {project!r} in targets metadata")
+        return str(max(stable))
+
+    def _fetch_targets_unverified(self) -> Targets:
+        """Fetch targets metadata without verifying TUF signatures.
+
+        Walks timestamp → snapshot → targets to resolve the current versioned
+        targets file and returns the parsed Targets signed payload.
+        """
+        base = self._repository_url
+
+        with urllib.request.urlopen(f'{base}/metadata/timestamp.json') as r:
+            ts: Metadata[Timestamp] = Metadata[Timestamp].from_bytes(r.read())
+        snap_ver = ts.signed.snapshot_meta.version
+
+        with urllib.request.urlopen(f'{base}/metadata/{snap_ver}.snapshot.json') as r:
+            snap: Metadata[Snapshot] = Metadata[Snapshot].from_bytes(r.read())
+        tgt_ver = snap.signed.meta['targets.json'].version
+
+        with urllib.request.urlopen(f'{base}/metadata/{tgt_ver}.targets.json') as r:
+            return Metadata[Targets].from_bytes(r.read()).signed
+
     # ------------------------------------------------------------------
     # Public API
     # ------------------------------------------------------------------
 
     def get_pointer(self, project: str, version: str | None = None) -> dict:
-        """Return the TUF-verified pointer JSON for *project* at *version*.
+        """Return the pointer JSON for *project* at *version*.
 
-        *version* = None resolves to ``latest.json`` (stable releases only).
+        Resolves *version* = None to the latest stable release by scanning
+        ``N.targets.json``.  The pointer file is fetched via its hash-prefixed
+        consistent-snapshot path.
         Raises ``TargetNotFoundError`` if the target is absent from the TUF repo.
         """
-        target_path = f'{project}/{version or "latest"}.json'
-
         with tempfile.TemporaryDirectory() as tmp:
             metadata_dir = Path(tmp) / 'metadata'
             target_dir = Path(tmp) / 'targets'
             metadata_dir.mkdir()
             target_dir.mkdir()
 
             if self._disable_verification:
-                url = f'{self._repository_url}/targets/{target_path}'
+                targets = self._fetch_targets_unverified()
+                resolved = self._resolve_version(project, version, targets)
+                target_path = f'{project}/{resolved}.json'
+
+                entry = targets.targets.get(target_path)
+                if entry is None:
+                    raise TargetNotFoundError(f'Target not found: {target_path}')
+
+                sha256 = entry.hashes.get('sha256', '')
+                url = f'{self._repository_url}/targets/{project}/{sha256}.{resolved}.json'
                 try:
                     with urllib.request.urlopen(url) as resp:
                         return json.loads(resp.read())
                 except urllib.error.HTTPError as exc:
                     if exc.code == 404:
-                        raise TargetNotFoundError(
-                            f'Pointer not found: {target_path}'
-                        ) from exc
+                        raise TargetNotFoundError(f'Pointer not found: {url}') from exc
                     raise
 
             self._bootstrap_metadata_dir(metadata_dir)
-
             updater = self._make_updater(metadata_dir, target_dir)
             updater.refresh()
 
+            # Parse the downloaded N.targets.json to resolve version and
+            # enumerate the hash-prefixed path the Updater will fetch.
+            candidates = sorted(metadata_dir.glob('*.targets.json'))
+            if not candidates:
+                raise TargetNotFoundError(f'No targets metadata after refresh for {project!r}')
+            targets = Metadata[Targets].from_file(str(candidates[-1])).signed
+
+            resolved = self._resolve_version(project, version, targets)
+            target_path = f'{project}/{resolved}.json'
+
             target_info = updater.get_targetinfo(target_path)
             if target_info is None:
-                raise TargetNotFoundError(
-                    f'No TUF target for {project!r} version {version or "latest"!r}'
-                )
+                raise TargetNotFoundError(f'No TUF target for {project!r} version {resolved!r}')
 
             pointer_path = target_dir / target_path
             pointer_path.parent.mkdir(parents=True, exist_ok=True)