Merge remote-tracking branch 'origin/master' into lucia-sb/remove-codecov

Author: lucia-sb

.builders/images/linux-aarch64/Dockerfile

@@ -36,11 +36,11 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS perl-core && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.13.12
+ENV PYTHON3_VERSION=3.13.13
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="12e7cb170ad2d1a69aee96a1cc7fc8de5b1e97a2bdac51683a3db016ec9a2996" \
+ SHA256="f9cde7b0e2ec8165d7326e2a0f59ea2686ce9d0c617dbbb3d66a7e54d31b74b9" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh \
  --prefix=/opt/python/${PYTHON3_VERSION} \

.builders/images/linux-aarch64/build_script.sh

@@ -23,6 +23,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
         VERSION="${kafka_version}" \
         SHA256="14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f" \
         RELATIVE_PATH="librdkafka-{{version}}" \
+        PATCHES="librdkafka-fix-coord-request-uaf.patch" \
         bash install-from-source.sh --enable-sasl --enable-curl --enable-zstd
     always_build+=("confluent-kafka")
 

.builders/images/linux-x86_64/Dockerfile

@@ -35,11 +35,11 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS perl-core && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.13.12
+ENV PYTHON3_VERSION=3.13.13
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="12e7cb170ad2d1a69aee96a1cc7fc8de5b1e97a2bdac51683a3db016ec9a2996" \
+ SHA256="f9cde7b0e2ec8165d7326e2a0f59ea2686ce9d0c617dbbb3d66a7e54d31b74b9" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh --prefix=/opt/python/${PYTHON3_VERSION} --with-ensurepip=yes --enable-ipv6 --with-dbmliborder=
 ENV PATH="/opt/python/${PYTHON3_VERSION}/bin:${PATH}"

.builders/images/linux-x86_64/build_script.sh

@@ -20,6 +20,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
         VERSION="${kafka_version}" \
         SHA256="14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f" \
         RELATIVE_PATH="librdkafka-{{version}}" \
+        PATCHES="librdkafka-fix-coord-request-uaf.patch" \
         bash install-from-source.sh --enable-sasl --enable-curl --enable-zstd
     always_build+=("confluent-kafka")
 

.builders/images/windows-x86_64/Dockerfile

@@ -85,11 +85,11 @@ RUN Get-RemoteFile `
     Approve-File -Path $($Env:USERPROFILE + '\.cargo\bin\rustc.exe') -Hash $Env:RUSTC_HASH
 
 # Install Python 3
-ENV PYTHON_VERSION="3.13.12"
+ENV PYTHON_VERSION="3.13.13"
 RUN Get-RemoteFile `
       -Uri https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION-amd64.exe `
       -Path python-$Env:PYTHON_VERSION-amd64.exe `
-      -Hash '96159fcb523ae404b707186a75b4104ee23851e476a5e838e14584cf1e03f981'; `
+      -Hash '3c9c81d80f91c002ced86d645422d81432c68c7d9b6b0e974768ca2e449a4d00'; `
     Start-Process -Wait python-$Env:PYTHON_VERSION-amd64.exe -ArgumentList '/quiet', 'InstallAllUsers=1'; `
     Remove-Item python-$Env:PYTHON_VERSION-amd64.exe; `
     & 'C:\Program Files\Python313\python.exe' -m pip install --no-warn-script-location --upgrade pip; `

.builders/patches/librdkafka-fix-coord-request-uaf.patch

@@ -0,0 +1,73 @@
+diff --git a/src/rdkafka_admin.c b/src/rdkafka_admin.c
+index 7986371a..552c10c0 100644
+--- a/src/rdkafka_admin.c
++++ b/src/rdkafka_admin.c
+@@ -480,7 +480,6 @@ rd_kafka_admin_coord_request(rd_kafka_broker_t *rkb,
+                              rd_kafka_replyq_t replyq,
+                              rd_kafka_resp_cb_t *resp_cb,
+                              void *opaque) {
+-        rd_kafka_t *rk             = rkb->rkb_rk;
+         rd_kafka_enq_once_t *eonce = opaque;
+         rd_kafka_op_t *rko;
+         char errstr[512];
+@@ -499,13 +498,17 @@ rd_kafka_admin_coord_request(rd_kafka_broker_t *rkb,
+             &rko->rko_u.admin_request.options, errstr, sizeof(errstr), replyq,
+             rd_kafka_admin_handle_response, eonce);
+ 
+-        if (err) {
+-                rd_kafka_admin_result_fail(
+-                    rko, err, "%s worker failed to send request: %s",
+-                    rd_kafka_op2str(rko->rko_type), errstr);
+-                rd_kafka_admin_common_worker_destroy(rk, rko,
+-                                                     rd_true /*destroy*/);
+-        }
++        /* On error, do not call worker_destroy() here.
++         * The caller (rd_kafka_coord_req_fsm) will call coord_req_fail()
++         * which enqueues a dummy error response carrying the eonce as
++         * opaque. coord_response_parse() will then retrieve the rko via
++         * del_source_return("coordinator response"), see the error, and
++         * call worker_destroy() itself.
++         *
++         * Calling worker_destroy() here would free the eonce while the
++         * caller still holds a reference to it, leading to a
++         * use-after-free / assertion failure in
++         * rd_kafka_enq_once_del_source_return(). */
+         return err;
+ }
+ 
+diff --git a/src/rdkafka_coord.c b/src/rdkafka_coord.c
+index a880f23a..e57b3ef7 100644
+--- a/src/rdkafka_coord.c
++++ b/src/rdkafka_coord.c
+@@ -316,6 +316,14 @@ static void rd_kafka_coord_req_fail(rd_kafka_t *rk,
+         rd_kafka_op_t *reply;
+         rd_kafka_buf_t *rkbuf;
+ 
++        if (!creq->creq_reply_opaque) {
++                /* The admin operation has already timed out and been
++                 * destroyed.  The eonce (creq_reply_opaque) may be
++                 * freed.  Do not enqueue a response referencing it. */
++                rd_kafka_coord_req_destroy(rk, creq, rd_true /*done*/);
++                return;
++        }
++
+         reply         = rd_kafka_op_new(RD_KAFKA_OP_RECV_BUF);
+         reply->rko_rk = rk; /* Set rk since the rkbuf will not have a rkb
+                              * to reach it. */
+@@ -495,7 +503,15 @@ static void rd_kafka_coord_req_fsm(rd_kafka_t *rk, rd_kafka_coord_req_t *creq) {
+                                                      replyq, creq->creq_resp_cb,
+                                                      creq->creq_reply_opaque);
+ 
+-                        if (err) {
++                        if (err == RD_KAFKA_RESP_ERR__DESTROY) {
++                                /* Admin op already timed out.  Clear
++                                 * opaque to prevent coord_req_fail from
++                                 * using freed eonce. */
++                                creq->creq_reply_opaque = NULL;
++                                rd_kafka_replyq_destroy(&replyq);
++                                rd_kafka_coord_req_destroy(rk, creq,
++                                                           rd_true /*done*/);
++                        } else if (err) {
+                                 /* Permanent error, e.g., request not
+                                  *  supported by broker. */
+                                 rd_kafka_replyq_destroy(&replyq);

.builders/promote.py

@@ -0,0 +1,121 @@
+"""Promote dependency wheels from dev to stable storage.
+
+Reads lockfiles from .deps/resolved/, identifies every wheel that lives
+under the ``dev/`` prefix in GCS, and copies it to the ``stable/`` prefix.
+Invoked via ``ddev promote <PR_URL>`` which dispatches the promote workflow.
+"""
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path, PurePosixPath
+
+from google.cloud import storage
+
+BUCKET_NAME = "deps-agent-int-datadoghq-com"
+REPO_DIR = Path(__file__).resolve().parent.parent
+LOCK_FILE_DIR = REPO_DIR / ".deps" / "resolved"
+
+DEV_PREFIX = "dev/"
+STABLE_PREFIX = "stable/"
+
+LOCKFILE_ENTRY = re.compile(
+    r"^(?P<name>\S+)\s+@\s+(?P<url>\S+)$"
+)
+
+
+def parse_lockfile_urls(lockfile: Path) -> list[str]:
+    """Extract wheel URLs from a lockfile."""
+    urls: list[str] = []
+    for line in lockfile.read_text().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        m = LOCKFILE_ENTRY.match(line)
+        if m:
+            urls.append(m.group("url").split("#")[0])
+    return urls
+
+
+STORAGE_BASE = "https://agent-int-packages.datadoghq.com/"
+STORAGE_TEMPLATE_PREFIX = f"{STORAGE_BASE}${{INTEGRATIONS_WHEELS_STORAGE}}/"
+
+
+def url_to_blob_path(url: str) -> str | None:
+    """Convert a wheel URL to its GCS blob path, or None if not a templated storage URL.
+
+    Handles the templated ``https://agent-int-packages.datadoghq.com/${INTEGRATIONS_WHEELS_STORAGE}/...``
+    format used in lockfiles.
+    """
+    if url.startswith(STORAGE_TEMPLATE_PREFIX):
+        return url[len(STORAGE_TEMPLATE_PREFIX):]
+    return None
+
+
+def collect_relative_paths() -> list[str]:
+    """Read all lockfiles and return relative wheel paths from ${INTEGRATIONS_WHEELS_STORAGE} entries."""
+    if not LOCK_FILE_DIR.is_dir():
+        print(f"No lockfile directory found at {LOCK_FILE_DIR}", file=sys.stderr)
+        sys.exit(1)
+
+    lockfiles = list(LOCK_FILE_DIR.glob("*.txt"))
+    if not lockfiles:
+        print(f"No lockfiles found in {LOCK_FILE_DIR}", file=sys.stderr)
+        sys.exit(1)
+
+    rel_paths: list[str] = []
+    for lockfile in sorted(lockfiles):
+        print(f"Reading {lockfile.name}")
+        for url in parse_lockfile_urls(lockfile):
+            rel_path = url_to_blob_path(url)
+            if rel_path:
+                rel_paths.append(rel_path)
+
+    return rel_paths
+
+
+def promote(rel_paths: list[str]) -> None:
+    """Copy blobs from dev/ to stable/ in GCS."""
+    if not rel_paths:
+        print("No templated wheels found in lockfiles — nothing to promote.")
+        return
+
+    unique_paths = sorted(set(rel_paths))
+    print(f"\nPromoting {len(unique_paths)} wheels from dev to stable...\n")
+
+    client = storage.Client()
+    bucket = client.bucket(BUCKET_NAME)
+
+    failed: list[str] = []
+    for rel_path in unique_paths:
+        dev_path = DEV_PREFIX + rel_path
+        stable_path = STABLE_PREFIX + rel_path
+        name = PurePosixPath(rel_path).name
+        source_blob = bucket.blob(dev_path)
+
+        if not source_blob.exists():
+            print(f"  MISSING  {name}")
+            failed.append(dev_path)
+            continue
+
+        bucket.copy_blob(source_blob, bucket, stable_path)
+        print(f"  OK       {name}")
+
+    print()
+    if failed:
+        print(
+            f"ERROR: {len(failed)} wheel(s) not found in dev storage.\n"
+            "The resolve-build-deps workflow may not have finished yet.\n"
+            "Wait for it to complete, then run ddev promote again.",
+            file=sys.stderr,
+        )
+        for p in failed:
+            print(f"  - {p}", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Done. {len(unique_paths)} wheel(s) promoted to stable.")
+
+
+if __name__ == "__main__":
+    rel_paths = collect_relative_paths()
+    promote(rel_paths)

.builders/scripts/files_to_remove.toml

@@ -17,30 +17,11 @@ excluded_paths = [
 
   # --- Tests ---
 
+  "tests/",
+  "/test/",
   "idlelib/idle_test/",
-  "bs4/tests/",
   "Cryptodome/SelfTest/",
-  "gssapi/tests/",
-  "keystoneauth1/tests/",
-  "lazy_loader/tests/",
-  "openstack/tests/",
-  "os_service_types/tests/",
-  "pbr/tests/",
-  "pkg_resources/tests/",
-  "pip/_vendor/colorama/tests/",
-  "psutil/tests/",
-  "requests_unixsocket/tests/",
   "securesystemslib/_vendor/ed25519/test_data/",
-  "setuptools/_distutils/compilers/C/tests/",
-  "setuptools/_vendor/packaging/tests/",
-  "setuptools/_distutils/tests/",
-  "setuptools/tests/",
-  "simplejson/tests/",
-  "stevedore/tests/",
-  "supervisor/tests/",
-  "/test/",
-  "vertica_python/tests/",
-  "websocket/tests/",
   "win32com/test/",
 
   # --- Botocore ---