perf: use serverless-specific wheel in layer (#17201)

## Description

This change splits the wheel build/validate/upload jobs into "regular" and "serverless" branches, for the purpose of allowing datadog-lambda-python to use custom-built wheels.

## Testing

CI passes

## Risks

Risk mitigated by keeping the existing wheel upload path working as usual while refactoring it.

## Additional Notes

Depended on by DataDog/datadog-lambda-python#762 and DataDog/serverless-tools#101

Co-authored-by: emmett.butler <emmett.butler@datadoghq.com>

Author: emmettbutler

.github/workflows/system-tests.yml

@@ -504,4 +504,4 @@ jobs:
           needs.integration-frameworks-system-tests.result == 'cancelled'|| 
           needs.tracer-release.result == 'failure' ||
           needs.tracer-release.result == 'cancelled'
-        run: exit 1
+        run: exit 1

.gitlab-ci.yml

@@ -125,9 +125,9 @@ serverless lambda tests:
   trigger:
     project: DataDog/datadog-lambda-python
     strategy: depend
-    branch: main
+    branch: main 
   needs:
-    - job: "upload manylinux2014"
+    - job: "upload serverless"
   variables:
     UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
     UPSTREAM_PROJECT_URL: $CI_PROJECT_URL

.gitlab/benchmarks/serverless.yml

@@ -5,6 +5,7 @@ benchmark-serverless:
   stage: benchmarks
   trigger:
     project: DataDog/serverless-tools
+    branch: main
     strategy: depend
   needs:
     - job: "upload serverless"

.gitlab/package.yml

@@ -197,16 +197,18 @@ variables:
 "upload serverless":
   extends: .upload_wheels_base
   needs:
-    - job: "build linux"
+    - job: "build linux serverless"
       artifacts: true
       parallel:
         matrix:
           - ARCH_TAG: "amd64"
-            PYTHON_TAG: "cp311-cp311"
-            IMAGE_TAG: "v85383392-751efc0-manylinux2014_x86_64"
+            PYTHON_TAG: *PYTHON_TAGS
+            IMAGE_TAG: *X86_64_IMAGES
+            DD_SERVERLESS_BUILD: "1"
           - ARCH_TAG: "arm64"
-            PYTHON_TAG: "cp311-cp311"
-            IMAGE_TAG: "v85383414-751efc0-manylinux2014_aarch64"
+            PYTHON_TAG: *PYTHON_TAGS
+            IMAGE_TAG: *AARCH64_IMAGES
+            DD_SERVERLESS_BUILD: "1"
     - job: "package version"
   variables:
     UPLOAD_SUFFIX: "serverless"
@@ -379,20 +381,32 @@ download_dependency_wheels:
       - "pywheels/*.whl"
 
 # Aggregate and validate all built wheels
-"ddtrace package":
+.package_base:
   image: registry.ddbuild.io/images/mirror/python:3.14.0
   tags: [ "arch:amd64" ]
   stage: package
+  before_script:
+    - pip install 'packaging>=24.0,<26'  # ci-deps: allow
+  artifacts:
+    paths:
+      - "pywheels/*.whl"
+      - "pywheels/*.tar.gz"
+
+"ddtrace package":
+  extends: .package_base
   needs:
     - "build linux"
     - "build macos"
     - "build sdist"
     - "collect windows wheels"
     - "package version"
   script:
-    - pip install 'packaging>=24.0,<26'  # ci-deps: allow
     - .gitlab/validate-ddtrace-package.py pywheels
-  artifacts:
-    paths:
-      - "pywheels/*.whl"
-      - "pywheels/*.tar.gz"
+
+"ddtrace package serverless":
+  extends: .package_base
+  needs:
+    - "build linux serverless"
+    - "package version"
+  script:
+    - .gitlab/validate-ddtrace-package.py pywheels --mode=serverless

.gitlab/release.yml

@@ -26,8 +26,7 @@ variables:
     - pip install -r ci/requirements/ci.txt
     - python -m twine check --strict pywheels/*
   script:
-    - rm -rf pywheels/ddtrace_serverless*
-    - python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/*
+    - python -m twine upload --repository ${PYPI_REPOSITORY} pywheels/ddtrace-*
   artifacts:
     paths:
       - pywheels/*.whl

.gitlab/validate-ddtrace-package.py

@@ -16,6 +16,7 @@
     PACKAGE_VERSION: Version from pyproject.toml (set by "package version" job)
 """
 
+import argparse
 import os
 from pathlib import Path
 import sys
@@ -41,21 +42,26 @@
 SERVERLESS_PLATFORMS = [p for p in BASE_PLATFORMS if "linux" in p]
 
 
-def build_expected_set(version: str) -> set[tuple[str, str, str]]:
+def build_expected_set(version: str, args: argparse.Namespace) -> set[tuple[str, str, str, str]]:
     """Build set of expected (version, python_tag, platform, flavor) tuples."""
     expected: set[tuple[str, str, str, str]] = set()
     for py_tag in PYTHON_TAGS:
-        for platform in BASE_PLATFORMS:
-            expected.add((version, py_tag, platform, ""))
-        # Add win_arm64 for Python 3.11+
-        if py_tag in WIN_ARM64_PYTHON_TAGS:
-            expected.add((version, py_tag, "win_arm64", ""))
+        if args.mode == "serverless":
+            for platform in SERVERLESS_PLATFORMS:
+                expected.add((version, py_tag, platform, "_serverless"))
+        else:
+            for platform in BASE_PLATFORMS:
+                expected.add((version, py_tag, platform, ""))
+            # Add win_arm64 for Python 3.11+
+            if py_tag in WIN_ARM64_PYTHON_TAGS:
+                expected.add((version, py_tag, "win_arm64", ""))
+
     return expected
 
 
 def reconstruct_wheel_filename(version: str, python_tag: str, platform: str, flavor: str) -> str:
     """Reconstruct wheel filename from components."""
-    package_name = f"ddtrace{flavor.replace('-', '_')}"
+    package_name = f"ddtrace{flavor}"
     return f"{package_name}-{version}-{python_tag}-{python_tag}-{platform}.whl"
 
 
@@ -89,7 +95,9 @@ def validate_sdist(wheels_dir: str, package_version: str) -> tuple[bool, str, st
         return False, f"Failed to parse sdist filename: {e}", sdist_path.name
 
 
-def parse_actual_wheels(wheels_dir: str) -> tuple[set[tuple[str, str, str, str]], list[str], int]:
+def parse_actual_wheels(
+    wheels_dir: str,
+) -> tuple[set[tuple[str, str, str, str]], list[str], int]:
     """Parse actual wheel files.
 
     Returns:
@@ -101,7 +109,6 @@ def parse_actual_wheels(wheels_dir: str) -> tuple[set[tuple[str, str, str, str]]
     for wheel_file in sorted(Path(wheels_dir).glob("*.whl")):
         try:
             name, version, build, tags = parse_wheel_filename(wheel_file.name)
-            flavor = name.replace("ddtrace", "").replace("_", "-")
             # Extract python tag - all tags should have the same interpreter
             py_tag = next(iter(tags)).interpreter
 
@@ -110,12 +117,13 @@ def parse_actual_wheels(wheels_dir: str) -> tuple[set[tuple[str, str, str, str]]
             # We know: name=ddtrace, abi=python tag (e.g., cp310)
             # So platform is everything after: ddtrace-{version}-{python}-{python}-
             wheel_base = wheel_file.name.replace(".whl", "")
-            marker = f"{name}-{version}-{py_tag}-{py_tag}-"
+            marker = f"{name.replace('-', '_')}-{version}-{py_tag}-{py_tag}-"
             if marker in wheel_base:
                 platform = wheel_base.split(marker)[1]
             else:
-                raise ValueError(f"Cannot parse platform from {wheel_file.name}")
+                raise ValueError(f"Cannot parse platform from {wheel_file.name} - searched for marker {marker}")
 
+            flavor = name.replace("ddtrace", "").replace("-", "_")
             actual.add((str(version), py_tag, platform, flavor))
         except Exception as e:
             errors.append(f"{wheel_file.name}: {e}")
@@ -124,7 +132,7 @@ def parse_actual_wheels(wheels_dir: str) -> tuple[set[tuple[str, str, str, str]]
 
 
 def identify_version_mismatches(
-    actual_set: set[tuple[str, str, str]], package_version: str
+    actual_set: set[tuple[str, str, str, str]], package_version: str
 ) -> dict[str, tuple[str, str]]:
     """Identify wheels with wrong versions."""
     mismatches: dict[str, tuple[str, str]] = {}
@@ -135,12 +143,9 @@ def identify_version_mismatches(
     return mismatches
 
 
-def main() -> None:
-    """Main validation function."""
-    # Get arguments
-    wheels_dir = sys.argv[1] if len(sys.argv) > 1 else "pywheels"
+def main(args: argparse.Namespace) -> None:
+    wheels_dir = args.wheels_dir
 
-    # Get version from environment
     package_version: str | None = os.environ.get("PACKAGE_VERSION")
     if not package_version:
         print("[ERROR] PACKAGE_VERSION not set. Ensure 'package version' job ran.")
@@ -167,15 +172,17 @@ def main() -> None:
     print()
 
     # Phase 2: SDist Validation
-    print("[Phase 2] SDist Validation")
-    sdist_ok, sdist_msg, sdist_name = validate_sdist(wheels_dir, package_version)
-    if not sdist_ok:
-        print(f"✗ {sdist_msg}")
-        errors.append(f"SDist validation: {sdist_msg}")
-    else:
-        print(f"✓ Found sdist: {sdist_name}")
-        print(f"✓ {sdist_msg}")
-    print()
+    sdist_ok = False
+    if args.mode != "serverless":
+        print("[Phase 2] SDist Validation")
+        sdist_ok, sdist_msg, sdist_name = validate_sdist(wheels_dir, package_version)
+        if not sdist_ok:
+            print(f"✗ {sdist_msg}")
+            errors.append(f"SDist validation: {sdist_msg}")
+        else:
+            print(f"✓ Found sdist: {sdist_name}")
+            print(f"✓ {sdist_msg}")
+        print()
 
     # Phase 3: Parse Actual Wheels
     print("[Phase 3] Parsing Actual Wheels")
@@ -194,7 +201,7 @@ def main() -> None:
 
     # Phase 4: Build Expected Set
     print("[Phase 4] Building Expected Set")
-    expected_set = build_expected_set(package_version)
+    expected_set = build_expected_set(package_version, args)
     print(f"Expected {len(expected_set)} wheels:")
     print(f"  - {len(PYTHON_TAGS)} Python versions (cp39-cp314)")
     print(f"  - {len(BASE_PLATFORMS)} base platforms")
@@ -281,4 +288,8 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    parser = argparse.ArgumentParser(prog="Validate DDTrace Package")
+    parser.add_argument("--mode", choices=["main", "serverless"], default="main")
+    parser.add_argument("wheels_dir", nargs="?", default="pywheels")
+    args = parser.parse_args()
+    main(args)