Merge branch 'master' into andy.anske/ocsf-transformations-zeek

Author: cepolation-datadog

.builders/build.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import json
 import os
 import shutil
 import subprocess
@@ -77,6 +78,8 @@ def build_macos():
                         help='Path to a folder where things will be installed during builder setup.')
     parser.add_argument('--skip-setup', default=False, action='store_true',
                         help='Skip builder setup, assuming it has already been set up.')
+    parser.add_argument('--constraints',
+                        help='Path to a pip constraints file for pinning transitive dependency versions.')
     args = parser.parse_args()
 
     image: str = args.image
@@ -108,6 +111,11 @@ def build_macos():
         shutil.copytree(HERE / 'scripts', mount_dir / 'scripts')
         shutil.copytree(HERE / 'patches', mount_dir / 'patches')
 
+        if args.constraints:
+            constraints_path = Path(args.constraints)
+            if constraints_path.is_file():
+                shutil.copy(constraints_path, mount_dir / 'constraints.txt')
+
         prefix_path = builder_root / 'prefix'
         env = {
             **os.environ,
@@ -153,6 +161,10 @@ def build_macos():
         final_requirements = mount_dir / 'frozen.txt'
         shutil.move(final_requirements, output_dir)
 
+        # Move the dependency sizes to the output directory
+        dependency_sizes_dir = mount_dir / 'sizes.json'
+        shutil.move(dependency_sizes_dir, output_dir)
+
 
 def build_image():
     parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
@@ -163,6 +175,8 @@ def build_image():
     parser.add_argument('--no-run', action='store_true')
     parser.add_argument('-a', '--build-arg', dest='build_args', nargs='+')
     parser.add_argument('-v', '--verbose', action='store_true')
+    parser.add_argument('--constraints',
+                        help='Path to a pip constraints file for pinning transitive dependency versions.')
     args = parser.parse_args()
 
     image: str = args.image
@@ -210,6 +224,11 @@ def build_image():
             shutil.copytree(HERE / 'scripts', mount_dir / 'scripts')
             shutil.copytree(HERE / 'patches', mount_dir / 'patches')
 
+            if args.constraints:
+                constraints_path = Path(args.constraints)
+                if constraints_path.is_file():
+                    shutil.copy(constraints_path, mount_dir / 'constraints.txt')
+
             # Create outputs on the host so they can be removed
             wheels_dir = mount_dir / 'wheels'
             wheels_dir.mkdir()
@@ -246,6 +265,9 @@ def build_image():
             # Move the final requirements file to the output directory
             shutil.move(final_requirements, output_dir)
 
+            # Move the dependency sizes to the output directory
+            shutil.move(mount_dir / 'sizes.json', output_dir)
+
 
 def main():
     if sys.platform == 'darwin':

.builders/deps/build_dependencies.txt

@@ -1,9 +1,10 @@
 hatchling==1.21.1
-setuptools==75.6.0
+hatch-requirements-txt
+setuptools==77.0.3
 wheel==0.38.4
 setuptools-scm
 setuptools-rust>=1.7.0
 maturin
 cffi>=1.12
-cython==3.0.11
+cython==3.2.1
 tomli>=2.0.1

.builders/images/install-from-source.sh

@@ -10,6 +10,7 @@
 # Optional:
 # - CONFIGURE_SCRIPT: Alternative to the default ./configure
 # - INSTALL_COMMAND: Specify a command for installation other than the default `make install`
+# - PATCHES: Paths to patches to apply, relative to the patches folder
 
 set -euxo pipefail
 
@@ -23,6 +24,11 @@ curl "${url}" -Lo "${workdir}/${archive_name}"
 echo "${SHA256}  ${workdir}/${archive_name}" | sha256sum --check
 tar -C "${workdir}" -xf "${workdir}/${archive_name}"
 pushd "${workdir}/${relative_path}"
+
+for p in ${PATCHES:-}; do
+    patch -p1 -i "${DD_MOUNT_DIR}/patches/${p}"
+done
+
 ${CONFIGURE_SCRIPT:-./configure} "$@"
 make -j $(nproc)
 ${INSTALL_COMMAND:-make install}

.builders/images/linux-aarch64/Dockerfile

@@ -16,11 +16,11 @@ ENV CXXFLAGS="${CFLAGS}"
 ENV LDFLAGS="-Wl,-rpath,'\$\$ORIGIN' -Wl,--strip-debug"
 
 # openssl
-RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
+RUN yum install -y perl-IPC-Cmd perl-CPANPLUS perl-core && \
  cpanp -i List::Util 1.66 && \
  DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
- VERSION="3.5.2" \
- SHA256="c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec" \
+ VERSION="3.6.2" \
+ SHA256="aaf51a1fe064384f811daeaeb4ec4dce7340ec8bd893027eee676af31e83a04f" \
  RELATIVE_PATH="openssl-{{version}}" \
  # https://docs.python.org/3/using/unix.html#custom-openssl
  INSTALL_COMMAND="make install_sw" \
@@ -36,11 +36,11 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.12.10
+ENV PYTHON3_VERSION=3.13.13
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="15d9c623abfd2165fe816ea1fb385d6ed8cf3c664661ab357f1782e3036a6dac" \
+ SHA256="f9cde7b0e2ec8165d7326e2a0f59ea2686ce9d0c617dbbb3d66a7e54d31b74b9" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh \
  --prefix=/opt/python/${PYTHON3_VERSION} \
@@ -110,7 +110,7 @@ RUN yum install -y flex && \
  VERSION="16.9" \
  SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
  RELATIVE_PATH="postgresql-{{version}}" \
- bash install-from-source.sh --without-readline --with-openssl --without-icu
+ bash install-from-source.sh --without-readline --with-openssl --without-icu --with-gssapi
 # Add paths to pg_config and to the library
 ENV PATH="/usr/local/pgsql/bin:${PATH}"
 ENV LD_LIBRARY_PATH="/usr/local/pgsql/lib/:${LD_LIBRARY_PATH}"
@@ -125,6 +125,9 @@ RUN \
  # This is the folder where unixODBC searches for driver config and where we ask customers to copy their config to
  --sysconfdir=/opt/datadog-agent/embedded/etc
 
+# zstd for librdkafka compression support
+RUN yum install -y epel-release && yum install -y libzstd-devel
+
 # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
 RUN \
  DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
@@ -154,8 +157,8 @@ RUN \
 # curl
 RUN \
  DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.16.0" \
- SHA256="a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970" \
+ VERSION="8.19.0" \
+ SHA256="2a2c11db4c122691aa23b4363befda1bfd801770bfebf41e1d21cee4f2ab0f71" \
  RELATIVE_PATH="curl-{{version}}" \
   bash install-from-source.sh \
     --disable-manual \

.builders/images/linux-aarch64/build_script.sh

@@ -21,9 +21,10 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
         VERSION="${kafka_version}" \
-        SHA256="a2c87186b081e2705bb7d5338d5a01bc88d43273619b372ccb7bb0d264d0ca9f" \
+        SHA256="14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f" \
         RELATIVE_PATH="librdkafka-{{version}}" \
-        bash install-from-source.sh --enable-sasl --enable-curl
+        PATCHES="librdkafka-fix-coord-request-uaf.patch" \
+        bash install-from-source.sh --enable-sasl --enable-curl --enable-zstd
     always_build+=("confluent-kafka")
 
     # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel

.builders/images/linux-x86_64/Dockerfile

@@ -16,11 +16,11 @@ ENV CXXFLAGS="${CFLAGS}"
 ENV LDFLAGS="-Wl,-rpath,'\$\$ORIGIN' -Wl,--strip-debug"
 
 # openssl
-RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
+RUN yum install -y perl-IPC-Cmd perl-CPANPLUS perl-core && \
  cpanp -i List::Util 1.66 && \
  DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
- VERSION="3.5.2" \
- SHA256="c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec" \
+ VERSION="3.6.2" \
+ SHA256="aaf51a1fe064384f811daeaeb4ec4dce7340ec8bd893027eee676af31e83a04f" \
  RELATIVE_PATH="openssl-{{version}}" \
  # https://docs.python.org/3/using/unix.html#custom-openssl
  INSTALL_COMMAND="make install_sw" \
@@ -35,11 +35,11 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.12.10
+ENV PYTHON3_VERSION=3.13.13
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="15d9c623abfd2165fe816ea1fb385d6ed8cf3c664661ab357f1782e3036a6dac" \
+ SHA256="f9cde7b0e2ec8165d7326e2a0f59ea2686ce9d0c617dbbb3d66a7e54d31b74b9" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh --prefix=/opt/python/${PYTHON3_VERSION} --with-ensurepip=yes --enable-ipv6 --with-dbmliborder=
 ENV PATH="/opt/python/${PYTHON3_VERSION}/bin:${PATH}"
@@ -114,7 +114,7 @@ RUN yum install -y flex && \
  VERSION="16.9" \
  SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
  RELATIVE_PATH="postgresql-{{version}}" \
- bash install-from-source.sh --without-readline --with-openssl --without-icu
+ bash install-from-source.sh --without-readline --with-openssl --without-icu --with-gssapi
 # Add paths to pg_config and to the library
 ENV PATH="/usr/local/pgsql/bin:${PATH}"
 ENV LD_LIBRARY_PATH="/usr/local/pgsql/lib/:${LD_LIBRARY_PATH}"
@@ -129,6 +129,9 @@ RUN \
  # This is the folder where unixODBC searches for driver config and where we ask customers to copy their config to
  --sysconfdir=/opt/datadog-agent/embedded/etc
 
+# zstd for librdkafka compression support
+RUN yum install -y epel-release && yum install -y libzstd-devel
+
 # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
 RUN \
  DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
@@ -158,8 +161,8 @@ RUN \
 # curl
 RUN \
  DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.16.0" \
- SHA256="a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970" \
+ VERSION="8.19.0" \
+ SHA256="2a2c11db4c122691aa23b4363befda1bfd801770bfebf41e1d21cee4f2ab0f71" \
  RELATIVE_PATH="curl-{{version}}" \
   bash install-from-source.sh \
     --disable-manual \

.builders/images/linux-x86_64/build_script.sh

@@ -18,9 +18,10 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
         VERSION="${kafka_version}" \
-        SHA256="a2c87186b081e2705bb7d5338d5a01bc88d43273619b372ccb7bb0d264d0ca9f" \
+        SHA256="14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f" \
         RELATIVE_PATH="librdkafka-{{version}}" \
-        bash install-from-source.sh --enable-sasl --enable-curl
+        PATCHES="librdkafka-fix-coord-request-uaf.patch" \
+        bash install-from-source.sh --enable-sasl --enable-curl --enable-zstd
     always_build+=("confluent-kafka")
 
     # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel

.builders/images/macos/builder_setup.sh

@@ -2,7 +2,6 @@
 
 set -euxo pipefail
 
-"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
 "${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
 "${DD_PYTHON3}" -m virtualenv py3
 
@@ -23,8 +22,8 @@ cp -R /opt/mqm "${DD_PREFIX_PATH}"
 
 # openssl
 DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
-VERSION="3.5.2" \
-SHA256="c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec" \
+VERSION="3.6.2" \
+SHA256="aaf51a1fe064384f811daeaeb4ec4dce7340ec8bd893027eee676af31e83a04f" \
 RELATIVE_PATH="openssl-{{version}}" \
 CONFIGURE_SCRIPT="./config" \
   install-from-source \
@@ -71,8 +70,8 @@ RELATIVE_PATH="libxslt-{{version}}" \
 
 # curl
 DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
-VERSION="8.16.0" \
-SHA256="a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970" \
+VERSION="8.19.0" \
+SHA256="2a2c11db4c122691aa23b4363befda1bfd801770bfebf41e1d21cee4f2ab0f71" \
 RELATIVE_PATH="curl-{{version}}" \
   install-from-source \
     --disable-manual \
@@ -103,6 +102,18 @@ RELATIVE_PATH="postgresql-{{version}}" \
 # Add paths to pg_config and to the library
 echo PATH="${DD_PREFIX_PATH}/bin:${PATH:-}" >> "$DD_ENV_FILE"
 
+# zstd for librdkafka compression support
+# Keep version in sync with github.com/DataDog/datadog-agent/deps/repos.MODULE.bazel
+DOWNLOAD_URL="https://github.com/facebook/zstd/releases/download/v{{version}}/zstd-{{version}}.tar.gz" \
+VERSION="1.5.7" \
+SHA256="eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" \
+RELATIVE_PATH="zstd-{{version}}" \
+CONFIGURE_SCRIPT="true" \
+INSTALL_COMMAND="make prefix=${DD_PREFIX_PATH} install" \
+  install-from-source
+# Fix install name so delocate can bundle it from the correct path
+install_name_tool -id "${DD_PREFIX_PATH}/lib/libzstd.1.dylib" "${DD_PREFIX_PATH}/lib/libzstd.1.dylib"
+
 # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
 DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
 VERSION="0.9.29" \

.builders/images/macos/extra_build.sh

@@ -13,13 +13,14 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
       VERSION="${kafka_version}" \
-      SHA256="a2c87186b081e2705bb7d5338d5a01bc88d43273619b372ccb7bb0d264d0ca9f" \
+      SHA256="14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f" \
       RELATIVE_PATH="librdkafka-{{version}}" \
-      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
+      bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl --enable-zstd
 
     # lmdb doesnt't get the actual full path in its install name which means delocate won't find it
     # Luckily we can patch it here so that it does.
-    install_name_tool -change liblmdb.so "${DD_PREFIX_PATH}/lib/liblmdb.so" "${DD_PREFIX_PATH}/lib//librdkafka.1.dylib"
+    install_name_tool -change liblmdb.so "${DD_PREFIX_PATH}/lib/liblmdb.so" "${DD_PREFIX_PATH}/lib/librdkafka.1.dylib"
+    install_name_tool -change /usr/local/lib/libzstd.1.dylib "${DD_PREFIX_PATH}/lib/libzstd.1.dylib" "${DD_PREFIX_PATH}/lib/librdkafka.1.dylib"
     always_build+=("confluent-kafka")
 fi
 

.builders/images/runner_dependencies.txt

@@ -1,5 +1,8 @@
 python-dotenv==1.0.0
-urllib3==2.2.0
+urllib3==2.6.3
 auditwheel==6.0.0; sys_platform == 'linux'
 delvewheel==1.5.2; sys_platform == 'win32'
 delocate==0.13.0; sys_platform == 'darwin'
+wheel==0.46.2
+packaging>=20.9  # Required by wheel>=0.46.0 (de-vendored)
+pathspec==0.12.1