Revert "refactor(downloader): resolve latest via S3 listing, drop latest.json reliance"

This reverts commit 70688d8.

Author: dkirov-dd

datadog_checks_downloader/datadog_checks/downloader/download_v2.py

@@ -6,7 +6,8 @@
 
 The v2 format stores a JSON pointer file as a TUF target at:
 
-    targets/<project>/<version>.json
+    targets/<project>/<version>.json   (versioned)
+    targets/<project>/latest.json      (latest stable, updated on each release)
 
 Each pointer file contains:
 
@@ -22,21 +23,6 @@
 The downloader TUF-verifies the pointer file, then fetches the wheel from
 ``repository + wheel_path`` and verifies the sha256 digest and byte length
 before writing to disk.
-
-Latest-version resolution
--------------------------
-
-When ``version`` is omitted, the downloader lists the bucket prefix
-``targets/<project>/`` directly via S3's ``ListObjectsV2`` REST API,
-filters keys to PEP 440 stable versions, and picks the maximum.  The
-chosen version is then fetched through TUF as usual, so the pointer file
-the client trusts is still cryptographically verified.
-
-Why list S3 instead of parsing the signed targets metadata: once
-``path_hash_prefixes`` delegations are in use in the TUF repo, the client
-cannot tell from the metadata alone which delegation signs the latest
-version of a given project.  A bucket listing sidesteps that — TUF still
-authoritatively verifies whichever version is then requested.
 """
 
 import hashlib
@@ -45,44 +31,15 @@
 import shutil
 import tempfile
 import urllib.error
-import urllib.parse
 import urllib.request
-import xml.etree.ElementTree as ET
 from pathlib import Path
 
-from packaging.version import InvalidVersion, Version
 from tuf.ngclient import Updater
 
 from .exceptions import DigestMismatch, TargetNotFoundError
 
 logger = logging.getLogger(__name__)
 
-# S3 ListObjectsV2 returns at most 1000 keys per request; pagination
-# continues via the ``NextContinuationToken`` in the response.  Constant is
-# the AWS-imposed max, included for clarity in the URL builder below.
-_S3_MAX_KEYS = 1000
-
-# All ListBucketResult elements live under this namespace.  Without a
-# namespace prefix in the XPath, ``ElementTree.find`` won't match them.
-_S3_NS = {'s3': 'http://s3.amazonaws.com/doc/2006-03-01/'}
-
-
-def _is_stable(version_str: str) -> bool:
-    """Return True if *version_str* is a stable PEP 440 release.
-
-    Stable means: parses as PEP 440, not a pre-release (aN/bN/rcN), not a
-    dev release (.devN), and no local-version identifier (+local).
-    Post-releases count as stable.  Mirrors the publisher's definition so
-    the two sides agree on what "stable" means.
-    """
-    try:
-        v = Version(version_str)
-    except InvalidVersion:
-        return False
-    if v.is_prerelease or v.is_devrelease:
-        return False
-    return v.local is None
-
 
 class TUFPointerDownloader:
     """Downloads Datadog integration wheels from a v2 TUF repository."""
@@ -115,7 +72,10 @@ def __init__(
         self._disable_verification = disable_verification
 
         if disable_verification:
-            logger.warning('Running with TUF verification disabled. Integrity is protected only by TLS (HTTPS).')
+            logger.warning(
+                'Running with TUF verification disabled. '
+                'Integrity is protected only by TLS (HTTPS).'
+            )
 
     # ------------------------------------------------------------------
     # Internal helpers
@@ -141,102 +101,17 @@ def _make_updater(self, metadata_dir: Path, target_dir: Path) -> Updater:
             target_dir=str(target_dir),
         )
 
-    def _list_s3_keys(self, prefix: str) -> list[str]:
-        """List every key under *prefix* in the public S3 bucket.
-
-        Uses S3's ListObjectsV2 REST API directly so we don't depend on
-        boto3.  Handles pagination via ``ContinuationToken``; returns all
-        keys under the prefix in arbitrary order (caller filters).
-
-        Raises ``TargetNotFoundError`` on HTTP 404 — that's how a missing
-        project surfaces (S3 returns the bucket but with KeyCount=0, but a
-        wholly unreachable bucket returns 404 / 403).
-        """
-        keys: list[str] = []
-        continuation_token: str | None = None
-        while True:
-            params = {
-                'list-type': '2',
-                'prefix': prefix,
-                'max-keys': str(_S3_MAX_KEYS),
-            }
-            if continuation_token is not None:
-                params['continuation-token'] = continuation_token
-            url = f'{self._repository_url}/?{urllib.parse.urlencode(params)}'
-            try:
-                with urllib.request.urlopen(url) as resp:
-                    body = resp.read()
-            except urllib.error.HTTPError as exc:
-                raise TargetNotFoundError(f'Cannot list {prefix} (HTTP {exc.code})') from exc
-
-            root = ET.fromstring(body)
-            for content in root.findall('s3:Contents', _S3_NS):
-                key_elem = content.find('s3:Key', _S3_NS)
-                if key_elem is not None and key_elem.text:
-                    keys.append(key_elem.text)
-
-            truncated_elem = root.find('s3:IsTruncated', _S3_NS)
-            if truncated_elem is None or truncated_elem.text != 'true':
-                break
-            token_elem = root.find('s3:NextContinuationToken', _S3_NS)
-            if token_elem is None or not token_elem.text:
-                # Truncated but no continuation token — treat as terminal
-                # rather than loop forever.  S3 should never produce this
-                # combination, but guard anyway.
-                break
-            continuation_token = token_elem.text
-        return keys
-
-    def _resolve_latest_version(self, project: str) -> str:
-        """List the bucket and return the highest PEP 440 stable version
-        for *project*.
-
-        Raises ``TargetNotFoundError`` if no stable version is published.
-        """
-        prefix = f'targets/{project}/'
-        keys = self._list_s3_keys(prefix)
-
-        # Each key under the prefix is ``targets/<project>/<version>.json``.
-        # Strip the prefix and the ``.json`` suffix to recover the version
-        # string, then filter to keys that look like ``<version>.json``
-        # (no further path segments — defensive against future siblings
-        # under ``targets/<project>/`` that aren't pointer files).
-        candidates: list[Version] = []
-        for key in keys:
-            tail = key[len(prefix) :]
-            if '/' in tail or not tail.endswith('.json'):
-                continue
-            version_str = tail[: -len('.json')]
-            if not _is_stable(version_str):
-                continue
-            try:
-                candidates.append(Version(version_str))
-            except InvalidVersion:
-                continue
-
-        if not candidates:
-            raise TargetNotFoundError(f'No stable version found under s3 prefix {prefix!r}')
-        return str(max(candidates))
-
     # ------------------------------------------------------------------
     # Public API
     # ------------------------------------------------------------------
 
     def get_pointer(self, project: str, version: str | None = None) -> dict:
         """Return the TUF-verified pointer JSON for *project* at *version*.
 
-        *version* = None resolves to the highest PEP 440 stable version
-        published under ``targets/<project>/`` in the bucket.  The chosen
-        version is then fetched through TUF, so the returned pointer is
-        still cryptographically verified.
-
-        Raises ``TargetNotFoundError`` if the target is absent from TUF or
-        no stable version is published.
+        *version* = None resolves to ``latest.json`` (stable releases only).
+        Raises ``TargetNotFoundError`` if the target is absent from the TUF repo.
         """
-        if version is None:
-            version = self._resolve_latest_version(project)
-
-        target_path = f'{project}/{version}.json'
+        target_path = f'{project}/{version or "latest"}.json'
 
         with tempfile.TemporaryDirectory() as tmp:
             metadata_dir = Path(tmp) / 'metadata'
@@ -251,7 +126,9 @@ def get_pointer(self, project: str, version: str | None = None) -> dict:
                         return json.loads(resp.read())
                 except urllib.error.HTTPError as exc:
                     if exc.code == 404:
-                        raise TargetNotFoundError(f'Pointer not found: {target_path}') from exc
+                        raise TargetNotFoundError(
+                            f'Pointer not found: {target_path}'
+                        ) from exc
                     raise
 
             self._bootstrap_metadata_dir(metadata_dir)
@@ -261,7 +138,9 @@ def get_pointer(self, project: str, version: str | None = None) -> dict:
 
             target_info = updater.get_targetinfo(target_path)
             if target_info is None:
-                raise TargetNotFoundError(f'No TUF target for {project!r} version {version!r}')
+                raise TargetNotFoundError(
+                    f'No TUF target for {project!r} version {version or "latest"!r}'
+                )
 
             pointer_path = target_dir / target_path
             pointer_path.parent.mkdir(parents=True, exist_ok=True)