feat: add llm-php-ext build support across platforms and update php version to 8.4.17 (#78)

Author: donhardman

.github/workflows/release.yml

@@ -13,7 +13,7 @@ on:
 env:
   APP_NAME: 'manticore-executor'
   DOCKER_TAG: 'manticoresearch/manticore-executor'
-  PHP_VERSION: '8.3.15'
+  PHP_VERSION: '8.4.17'
   MAINTAINER: 'Manticore'
   DESC: 'Custom built PHP executor for Manticore.'
   EXTRA_NAME: 'manticore-extra'
@@ -103,12 +103,12 @@ jobs:
             use_buildx: true
             run: ""
           - name: AMD64 native on OSX
-            os: macos-13
+            os: macos-15-intel
             arch: x86_64
             use_buildx: false
             run: ./build-osx
           - name: ARM64 native on OSX
-            os: macos-14
+            os: macos-15
             arch: arm64
             use_buildx: false
             run: ./build-osx
@@ -557,7 +557,7 @@ jobs:
           - artifact_suffix: ubuntu-22.04-arm64
             asset_name_suffix: linux-arm64.tar.gz
             asset_content_type: application/gzip
-          - artifact_suffix: macos-13-x86_64
+          - artifact_suffix: macos-15-x86_64
             asset_name_suffix: macos-x86_64.tar.gz
             asset_content_type: application/gzip
           - artifact_suffix: macos-14-arm64
@@ -811,7 +811,7 @@ jobs:
             arch: aarch
             delimiter: "_"
             distro: 10
-          - artifact: artifact-macos-13-x86_64
+          - artifact: artifact-macos-15-x86_64
             type: arc
             arch: x86_64
             delimiter: "-"

.github/workflows/test.yml

@@ -7,7 +7,7 @@ on:
 
 env:
   APP_NAME: 'manticore-executor'
-  PHP_VERSION: '8.3.15'
+  PHP_VERSION: '8.4.17'
   MAINTAINER: 'Manticore'
   DESC: 'Custom built PHP executor for Manticore.'
 
@@ -30,13 +30,13 @@ jobs:
             arch: amd64
             build_type: jammy
           - job_name: 'Build static on amd64 OSX'
-            os: macos-13
+            os: macos-15-intel
             use_docker: false
             script_name: ./build-osx
             arch: amd64
             build_type: osx
           - job_name: 'Build static on arm64 OSX'
-            os: macos-14
+            os: macos-15
             use_docker: false
             script_name: ./build-osx
             arch: arm64
@@ -52,7 +52,7 @@ jobs:
         run: ${{ matrix.script_name }} ${{ env.PHP_VERSION }}
       - name: Build binary using Docker
         if: ${{ matrix.use_docker == true }}
-        run: docker run -v ${PWD}:/src -w /src alpine:3.17 sh -c "apk add sudo bash; ${{ matrix.script_name }} ${{ env.PHP_VERSION }}"
+        run: docker run -v ${PWD}:/src -w /src alpine:3.21 sh -c "apk add sudo bash; ${{ matrix.script_name }} ${{ env.PHP_VERSION }}"
       - name: Create artifact
         run: |
           os="${{ runner.os }}"

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.17
+FROM alpine:3.21
 ARG BUILD_DEV=0
 
 # Install the manticore-executor binary
 COPY . /src
 RUN apk add bash && \
   cd /src && \
-  ./build-alpine 8.4.2 0 $BUILD_DEV && \
+  ./build-alpine 8.4.17 0 $BUILD_DEV && \
   mv build/dist/bin/php /usr/bin/manticore-executor && \
   ln -s /usr/bin/manticore-executor /usr/bin/php && \
   cd ../..

Dockerfile-dist

@@ -6,7 +6,7 @@ RUN apt-get update -y && \
 COPY ./build-linux ./build-linux
 COPY ./build-bash-base.sh ./build-bash-base.sh
 COPY ./helper.sh ./helper.sh
-RUN ./build-linux 8.4.2 0 0 && \
+RUN ./build-linux 8.4.17 0 0 && \
   cp build/dist/bin/php /usr/bin/manticore-executor && \
   rm -fr build
 

README.md

@@ -28,7 +28,7 @@ Manticore Executor is a custom built PHP binary which:
 
 By default, we disable all extensions and enable only those required to run our scripts.
 
-We build executor from `PHP 8.4.2` with the following extensions enabled and compiled into the executable statically:
+We build executor from `PHP 8.4.17` with the following extensions enabled and compiled into the executable statically:
 
 * pcntl
 * posix
@@ -83,10 +83,10 @@ Several scripts used to build the final package:
 * `build-linux`
 * `build-osx`
 
-The scripts accept a version of PHP as a parameter. The current version is `8.4.2`. To build the binary, you should run the following example:
+The scripts accept a version of PHP as a parameter. The current version is `8.4.17`. To build the binary, you should run the following example:
 
 ```bash
-./build-linux "8.4.2"
+./build-linux "8.4.17"
 ```
 
-The command above will build the package on Linux with **PHP** `8.4.2`. Once it's done, you can find your binary in folder `dist/bin`.
+The command above will build the package on Linux with **PHP** `8.4.17`. Once it's done, you can find your binary in folder `dist/bin`.

build-alpine

@@ -22,7 +22,12 @@ install_deps() {
     ln -s /usr/bin/objcopy "/usr/bin/${arch}-alpine-linux-musl-objcopy"
   export CROSS_COMPILE="/usr/bin/${arch}-alpine-linux-musl-"
 
-  apk add curl git autoconf bison re2c pkgconf zlib-dev zstd-dev openssl-libs-static openssl-dev zlib-static zlib-dev gnu-libiconv-dev
+  apk add php84 php84-dev curl git autoconf bison re2c pkgconf zlib-dev zstd-static zstd-dev openssl-libs-static openssl-dev zlib-static zlib-dev gnu-libiconv-dev clang-dev llvm-dev
+
+  # Create symlinks for php84 to be accessible as php
+  ln -sf /usr/bin/php84 /usr/bin/php
+  ln -sf /usr/bin/php-config84 /usr/bin/php-config
+  ln -sf /usr/bin/phpize84 /usr/bin/phpize
 
   install_curl
 
@@ -31,6 +36,8 @@ install_deps() {
   install_zlib
 
   install_rdkafka
+
+  install_rust
 }
 
 install_dev_deps() {
@@ -39,7 +46,7 @@ install_dev_deps() {
   install_libxml2
 }
 
-export -f install_deps install_dev_deps
+export -f install_deps install_dev_deps install_rust
 . build-bash-base.sh "$@"
 
 if [ "$BUILD_DEV" -eq 1 ]; then
@@ -61,6 +68,7 @@ ENABLE_STATIC=yes
   --with-rdkafka \
 	--enable-jchash \
 	--enable-simdjson \
+	--enable-llm \
   --with-openssl --with-zlib --with-zip \
   --enable-filter \
   --with-curl \
@@ -77,3 +85,8 @@ fix_static_linking
 
 make LDFLAGS=$MAKE_LDFLAGS -j8
 make install-cli
+
+# Remove php84 that was used for building before installing php
+apk del php84 php84-dev
+rm -f /usr/bin/php /usr/bin/php-config /usr/bin/phpize
+

build-bash-base.sh

@@ -8,12 +8,14 @@
 set -e
 
 PHP_VERSION="$1"
-ZSTD_REV="2dfcd6524ccdcef6dfdaa97d7f3716b866885093"
-DS_REV="da4d2f2a2c0f3732b34562636849c5e52e79e6c3"
-SWOOLE_REV="3e1a1f89930ba0bbea1f5ee31bcd0ee701a87aab"
+ZSTD_REV="d6d26eb805058fc672686715a7b5e39d8f65dd36"
+DS_REV="118c06b8863386ceada4238f3cec18ab84c9efb7"
+SWOOLE_REV="d611ff3ca60e8c9425b2e2e3ba1a9004faab64f5"
 RDKAFKA_REV="53398031f1cd96e437e9705b67b4dc19d955acb6"
+
 JCHASH_REV="8ed50cc8c211effe1c214eae1e3240622e0f11b0"
 SIMDJSON_REV="9a2745669fea733a40f9443b1a793846d0759b89"
+LLM_REV="8ef13fa0911de91fddebe0bde48ba7569a055482"
 SKIP_SYSTEM_DEPS="$2"
 BUILD_DEV="$3"
 BUILD_STATIC=1 # Always build static but dev
@@ -79,7 +81,22 @@ else
 fi
 cd ..
 
-cd ..
+# llm
+git clone https://github.com/manticoresoftware/llm-php-ext.git llm
+cd llm && git checkout "$LLM_REV"
+if [ "$(uname)" == "Darwin" ]; then
+	# macOS: Use LLVM 17 from Homebrew
+	LLVM_PREFIX="$(brew --prefix llvm@17 2>/dev/null || echo /opt/homebrew/opt/llvm@17)"
+	LIBCLANG_PATH="$LLVM_PREFIX/lib"
+	LLVM_CONFIG_PATH="$LLVM_PREFIX/bin/llvm-config"
+	LLVM_PATH="$LLVM_PREFIX/bin"
+	# Ensure PHP is still accessible by appending LLVM to PATH instead of prepending
+	export LIBCLANG_PATH="$LIBCLANG_PATH" LLVM_CONFIG_PATH="$LLVM_CONFIG_PATH" PATH="$PATH:$LLVM_PATH"
+	# Verify PHP is accessible
+	echo "Verifying PHP is accessible for ext-php-rs build..."
+	which php && php --version || echo "WARNING: php not found in PATH!"
+fi
+cd ../..
 
 BUILD_EXTRA=()
 if [[ "$BUILD_DEV" == "1" ]]; then
@@ -102,7 +119,7 @@ if [[ "$BUILD_DEV" == "1" ]]; then
     # "--enable-memprof"
     # "--enable-memprof-debug"
 		#
-		# It does not work with PHP 8.4.2
+		# It does not work with PHP 8.4.17
     # "--enable-tideways-xhprof"
     # "--enable-xdebug"
   )

build-linux

@@ -17,14 +17,24 @@ set -xe
 
 install_deps() {
   sudo apt-get update -y
-  sudo apt-get install -y curl build-essential autoconf automake bison re2c libzstd-dev pkg-config git libssl-dev zlib1g-dev libzip-dev libonig-dev cmake libcurl4-openssl-dev libzip-dev librdkafka-dev libiconv-hook-dev
+  sudo apt-get install -y software-properties-common
+  sudo add-apt-repository ppa:ondrej/php -y
+  sudo apt-get update -y
+  sudo apt-get install -y php8.4 php8.4-dev php8.4-cli build-essential clang libclang-dev pkg-config curl autoconf automake bison re2c libzstd-dev git libssl-dev zlib1g-dev libzip-dev libonig-dev cmake libcurl4-openssl-dev librdkafka-dev libiconv-hook-dev wget
+  # Install LLVM 17 directly from apt.llvm.org
+  wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg
+  echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" | sudo tee /etc/apt/sources.list.d/llvm.list > /dev/null
+  sudo apt-get update -y
+  sudo apt-get install -y clang-17 lldb-17 lld-17 clangd-17 clang-tidy-17 clang-format-17 clang-tools-17 llvm-17-dev libomp-17-dev libc++-17-dev libc++abi-17-dev libclang-common-17-dev libclang-17-dev libclang-cpp17-dev || echo "LLVM 17 packages not all available, continuing..."
+
+  install_rust
 }
 
 install_dev_deps() {
   sudo apt-get install -y libonig-dev libgmp-dev libxml2-dev
 }
 
-export -f install_deps install_dev_deps
+export -f install_deps install_dev_deps install_rust
 . build-bash-base.sh "$@"
 
 
@@ -47,6 +57,7 @@ ENABLE_STATIC=yes
   --with-rdkafka \
 	--enable-jchash \
 	--enable-simdjson \
+	--enable-llm \
   --with-openssl --with-zlib --with-zip --with-iconv \
   --enable-filter \
   --with-curl \
@@ -59,3 +70,7 @@ ENABLE_STATIC=yes
 
 make -j8
 make install-cli
+
+# Remove php84 that was used for building before installing php
+sudo apt-get remove -y php8.4-dev php8.4-cli php8.4
+

build-osx

@@ -7,8 +7,17 @@
 # you can find it at https://www.php.net/license/3_01.txt
 
 set -e
+. helper.sh
+
 install_deps() {
-  brew install bash curl autoconf automake bison re2c zstd git openssl zlib cmake libzip librdkafka libiconv
+  brew tap shivammathur/php
+  brew install shivammathur/php/php@8.4 bash curl autoconf automake bison re2c zstd git openssl zlib cmake libzip librdkafka libiconv llvm@17
+  brew link --overwrite --force shivammathur/php/php@8.4
+
+  # Add PHP to PATH - include both versioned path and brew bin directory
+  PHP_PREFIX="$(brew --prefix shivammathur/php/php@8.4)"
+  BREW_PREFIX="$(brew --prefix)"
+  export PATH="${PHP_PREFIX}/bin:${BREW_PREFIX}/bin:${PATH}"
 
   OPENSSL_PREFIX="$(brew --prefix openssl)"
   LIBRDKAFKA_PREFIX="$(brew --prefix librdkafka)"
@@ -21,42 +30,17 @@ install_deps() {
   export CPPFLAGS="-I${OPENSSL_PREFIX}/include -I${LIBRDKAFKA_PREFIX}/include -I${ICONV_PREFIX}/include"
 
   export PKG_CONFIG_PATH="${OPENSSL_PREFIX}/lib/pkgconfig"
+
+  install_rust
 }
 
 install_dev_deps() {
   brew install libxml2 oniguruma libgmp
 }
 
-export -f install_deps install_dev_deps
+export -f install_deps install_dev_deps install_rust
 . ./build-bash-base.sh "$@"
 
-hide_dynamic_libs() {
-  ! which brew && return
-  for lib in "$@"; do
-    LIB_PATH=$(brew --prefix "$lib")
-    if ls "$LIB_PATH"/lib/*.dylib; then
-      for f in "$LIB_PATH"/lib/*.dylib; do
-        mv "$f" "$f.bak"
-      done
-    fi
-  done
-}
-
-restore_dynamic_libs() {
-  ! which brew && return
-  for lib in "$@"; do
-    LIB_PATH=$(brew --prefix "$lib")
-    if ls "$LIB_PATH"/lib/*.dylib.bak; then
-      for f in "$LIB_PATH"/lib/*.dylib.bak; do
-        mv "$f" "${f%*.bak}"
-      done
-    fi
-  done
-}
-
-# Leave only static libs on
-hide_dynamic_libs zstd zlib
-
 MAKE_LDFLAGS=
 ENABLE_SHARED=yes
 ENABLE_STATIC=no
@@ -95,6 +79,7 @@ chmod +x configure
   --with-rdkafka \
 	--enable-jchash \
 	--enable-simdjson \
+	--enable-llm \
   --with-openssl --with-zlib --with-zip --with-iconv=$(brew --prefix libiconv) \
   --enable-filter \
   --with-curl \
@@ -109,5 +94,3 @@ make LDFLAGS=$MAKE_LDFLAGS -j8
 make install-cli
 strip dist/bin/php
 
-# Return skipped dynamic libs
-restore_dynamic_libs zstd zlib

helper.sh

@@ -1,6 +1,13 @@
 #!/usr/bin/env bash
 set -ex
 
+install_rust() {
+	# Install Rust toolchain
+	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+	# Source cargo environment
+	. "$HOME/.cargo/env"
+}
+
 install_curl() {
 	test -d curl-8.0.0 && rm -fr "$_"
 	curl -sSL https://github.com/curl/curl/releases/download/curl-8_0_0/curl-8.0.0.tar.gz | tar -xzf -
@@ -64,7 +71,7 @@ build_dev_conf() {
 	git checkout 68eb143bd5700a6fe041826118aeb9a13a3fcef3
 	cd ..
 
-	# It does not work with PHP 8.4.2
+	# It does not work with PHP 8.4.17
 	# git clone https://github.com/tideways/php-xhprof-extension.git tideways_xhprof && cd "$_"
 	# git checkout 7877082945fcccced12676749676af1736a00f65
 	# cd ..
@@ -84,4 +91,4 @@ fix_static_linking() {
 	sed -ie 's/PHP_INI_ENTRY("openssl.cafile", NULL, PHP_INI_PERDIR, NULL)/PHP_INI_ENTRY("openssl.cafile", "\/etc\/ssl\/cert.pem", PHP_INI_PERDIR, NULL)/g' ext/openssl/openssl.c
 	sed -ie 's/-export-dynamic//g' Makefile
 	sed -ie 's/-o $(SAPI_CLI_PATH)/-all-static -o $(SAPI_CLI_PATH)/g' Makefile
-}
+}