wip move to communication via sidecar

Author: cataphract

.gitlab-ci.yml

@@ -82,7 +82,7 @@ appsec-trigger:
     strategy: depend
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
+    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/libddwaf-rust
 
 profiler-trigger:
   stage: tests
@@ -119,7 +119,7 @@ package-trigger:
       pipeline_variables: true
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
+    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/libddwaf-rust
     NIGHTLY_BUILD: $NIGHTLY_BUILD
 
 # Runs after the full CI completes.  Triggered in two situations:

.gitlab/build-appsec-helper-rust.sh

@@ -22,7 +22,7 @@ cargo +nightly-"$RUST_TARGET" build \
     --target "$RUST_TARGET"
 
 # Remove musl libc dependency using patchelf (makes binary work on both musl and glibc)
-BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper_rust.so"
+BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper.so"
 ARCH=$(uname -m)
 if [ "$ARCH" = "x86_64" ]; then
     patchelf --remove-needed libc.musl-x86_64.so.1 "$BINARY_PATH" 2>/dev/null || true
@@ -31,7 +31,8 @@ elif [ "$ARCH" = "aarch64" ]; then
 fi
 
 # Copy to output
-cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper-rust.so"
+cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper.so"
+cp -v ../recommended.json "../../appsec_$(uname -m)/"
 
 # Run tests
 cargo +nightly-"$RUST_TARGET" test \

.gitlab/build-appsec-helper.sh

@@ -13,17 +13,15 @@ suffix="${1:-}"
 echo "Build nts extension"
 switch-php "${PHP_VERSION}"
 mkdir -p appsec/build ; cd appsec/build
-cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_BUILD_HELPER=OFF \
-  -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
+cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
 make -j $MAKE_JOBS
 cp -v ddappsec.so "../../appsec_$(uname -m)/ddappsec-$PHP_API${suffix}.so"
 cd "../../"
 
 echo "Build zts extension"
 switch-php "${PHP_VERSION}-zts"
 mkdir -p appsec/build-zts ; cd appsec/build-zts
-cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_BUILD_HELPER=OFF \
-  -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
+cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
 make -j $MAKE_JOBS
 cp -v ddappsec.so "../../appsec_$(uname -m)/ddappsec-$PHP_API${suffix}-zts.so"
 cd "../../"

.gitlab/build-appsec.sh

@@ -112,7 +112,6 @@
     KUBERNETES_MEMORY_LIMIT: 30Gi
     DOCKER_LOOPBACK_SIZE: 30G
     ARCH: amd64
-    HELPER_RUST_FLAG: ""
     GRADLE_USER_HOME: "$CI_PROJECT_DIR/.gradle-home"
     DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED: "0"
   before_script:
@@ -129,7 +128,7 @@
         TERM=dumb ./gradlew loadCaches --info
       fi
 
-      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps $HELPER_RUST_FLAG
+      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps
       TERM=dumb ./gradlew saveCaches --info
   after_script:
     - mkdir -p "${CI_PROJECT_DIR}/artifacts"
@@ -183,17 +182,6 @@
       - targets:
           - test8.3-release-ssi
 
-"appsec integration tests (helper-rust)":
-  extends: .appsec_integration_tests
-  variables:
-    HELPER_RUST_FLAG: "-PuseHelperRust"
-  parallel:
-    matrix:
-      - targets:
-          - test7.4-release
-          - test8.1-release
-          - test8.3-debug
-
 "helper-rust build and test":
   stage: test
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
@@ -419,11 +407,6 @@
       export PATH=$PATH:$HOME/.cargo/bin
       LLVM_PROFILE_FILE="/tmp/cov-ext/%p.profraw" \
         VERBOSE=1 make -j 4 xtest
-    - VERBOSE=1 make -j 4 ddappsec_helper_test
-    - |
-      cd ../..
-      LLVM_PROFILE_FILE="/tmp/cov-helper/%p.profraw" \
-        ./appsec/build/tests/helper/ddappsec_helper_test
     - |
       cd /tmp/cov-ext
       llvm-profdata-17 merge -sparse *.profraw -o default.profdata
@@ -433,15 +416,6 @@
       echo "Uploading extension coverage to codecov"
       cd "$CI_PROJECT_DIR"
       codecov -t "$CODECOV_TOKEN" -n appsec-extension -v -f appsec/build/coverage-ext.lcov
-    - |
-      cd /tmp/cov-helper
-      llvm-profdata-17 merge -sparse *.profraw -o default.profdata
-      llvm-cov-17 export "$CI_PROJECT_DIR"/appsec/build/tests/helper/ddappsec_helper_test \
-        -format=lcov -instr-profile=default.profdata \
-        > "$CI_PROJECT_DIR/appsec/build/coverage-helper.lcov"
-      echo "Uploading helper coverage to codecov"
-      cd "$CI_PROJECT_DIR"
-      codecov -t "$CODECOV_TOKEN" -n appsec-helper -v -f appsec/build/coverage-helper.lcov
     - |
       echo "Uploading coverage to Datadog"
       cd "$CI_PROJECT_DIR"
@@ -515,84 +489,9 @@
         -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" \
         -DCLANG_TIDY=/usr/bin/run-clang-tidy-17 \
         -DCLANG_FORMAT=/usr/bin/clang-format-17
-    - make -j 4 extension ddappsec-helper
+    - make -j 4 extension
     - make format tidy
 
-"test appsec helper asan":
-  stage: test
-  extends: .appsec_test
-  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-6
-  variables:
-    KUBERNETES_CPU_REQUEST: 3
-    KUBERNETES_MEMORY_REQUEST: 3Gi
-    KUBERNETES_MEMORY_LIMIT: 4Gi
-  parallel:
-    matrix:
-      - ARCH: *arch_targets
-  script:
-    - cd appsec/build
-    - |
-      cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF \
-        -DDD_APPSEC_ENABLE_COVERAGE=OFF -DDD_APPSEC_TESTING=ON \
-        -DCMAKE_CXX_FLAGS="-stdlib=libc++ -fsanitize=address -fsanitize=leak \
-        -DASAN_BUILD" -DCMAKE_C_FLAGS="-fsanitize=address -fsanitize=leak \
-        -DASAN_BUILD" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-        -DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-        -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" \
-        -DCLANG_TIDY=/usr/bin/run-clang-tidy-17
-    - make -j 4 ddappsec_helper_test
-    - cd ../..; ./appsec/build/tests/helper/ddappsec_helper_test
-
-### Disabled: "we don't rely on the fuzzer these days as the protocol has been stable for a long time, so feel free to disable those jobs for now"
-#"fuzz appsec helper":
-#  stage: test
-#  extends: .appsec_test
-#  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-6
-#  variables:
-#    KUBERNETES_CPU_REQUEST: 3
-#    KUBERNETES_MEMORY_REQUEST: 5Gi
-#    KUBERNETES_MEMORY_LIMIT: 6Gi
-#  parallel:
-#    matrix:
-#      - ARCH: *arch_targets
-#  script:
-#    - curl -LO https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-17.0.6.tar.gz
-#    - tar xzf llvmorg-17.0.6.tar.gz
-#    - cd llvm-project-llvmorg-17.0.6/compiler-rt
-#    - cmake . -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_CXX_LINK_FLAGS="-stdlib=libc++"
-#    - make -j 4 fuzzer
-#    - fuzzer=$(realpath lib/linux/libclang_rt.fuzzer_no_main-*.a)
-#    - cd -
-#
-#    - cd appsec/build
-#    - cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_CXX_LINK_FLAGS="-stdlib=libc++" -DFUZZER_ARCHIVE_PATH=$fuzzer -DBOOST_CACHE_PREFIX=/boost-cache -DCLANG_TIDY=/usr/bin/run-clang-tidy-17
-#    - make -j 4 ddappsec_helper_fuzzer corpus_generator
-#    - cd ..
-#    - mkdir -p tests/fuzzer/{corpus,results,logs}
-#    - rm -f tests/fuzzer/corpus/*
-#
-#    - '# Run fuzzer in nop mode'
-#    - ./build/tests/fuzzer/corpus_generator tests/fuzzer/corpus 500
-#    - LLVM_PROFILE_FILE=off.profraw ./build/tests/fuzzer/ddappsec_helper_fuzzer --log_level=off --fuzz-mode=off -max_total_time=60 -rss_limit_mb=4096 -artifact_prefix=tests/fuzzer/results/ tests/fuzzer/corpus/
-#    - rm -f tests/fuzzer/corpus/*
-#
-#    - '# Run fuzzer in raw mode'
-#    - ./build/tests/fuzzer/corpus_generator tests/fuzzer/corpus 500
-#    - LLVM_PROFILE_FILE=raw.profraw ./build/tests/fuzzer/ddappsec_helper_fuzzer --log_level=off --fuzz-mode=raw -max_total_time=60 -rss_limit_mb=4096 -artifact_prefix=tests/fuzzer/results/ tests/fuzzer/corpus/
-#    - rm -f tests/fuzzer/corpus/*
-#
-#    - '# Run fuzzer in body mode'
-#    - ./build/tests/fuzzer/corpus_generator tests/fuzzer/corpus 500
-#    - LLVM_PROFILE_FILE=body.profraw ./build/tests/fuzzer/ddappsec_helper_fuzzer --log_level=off --fuzz-mode=body -max_total_time=60 -rss_limit_mb=4096 -artifact_prefix=tests/fuzzer/results/ tests/fuzzer/corpus/
-#
-#    - '# Generate coverage'
-#    - llvm-profdata-17 merge -sparse *.profraw -o default.profdata
-#    - llvm-cov-17 show build/tests/fuzzer/ddappsec_helper_fuzzer -instr-profile=default.profdata -ignore-filename-regex="(tests|third_party|build)" -format=html > fuzzer-coverage.html
-#    - llvm-cov-17 report -instr-profile default.profdata build/tests/fuzzer/ddappsec_helper_fuzzer -ignore-filename-regex="(tests|third_party|build)" -show-region-summary=false
-#  artifacts:
-#    paths:
-#     - appsec/fuzzer-coverage.html
-
 "check libxml2 version":
   stage: test
   image: registry.ddbuild.io/images/mirror/python:3.12-slim-bullseye

.gitlab/generate-appsec.php

@@ -283,24 +283,6 @@
 }
 ?>
 
-"compile appsec helper":
-  stage: appsec
-  image: "registry.ddbuild.io/images/mirror/b1o7r7e0/nginx_musl_toolchain"
-  tags: [ "arch:$ARCH" ]
-  needs: [ "prepare code" ]
-  parallel:
-    matrix:
-      - ARCH: ["amd64", "arm64" ]
-  variables:
-    MAKE_JOBS: 12
-    KUBERNETES_CPU_REQUEST: 12
-    KUBERNETES_MEMORY_REQUEST: 4Gi
-    KUBERNETES_MEMORY_LIMIT: 8Gi
-  script: .gitlab/build-appsec-helper.sh
-  artifacts:
-    paths:
-      - "appsec_*"
-
 "compile appsec helper rust":
   stage: appsec
   image: "registry.ddbuild.io/images/mirror/datadog/dd-appsec-php-ci:nginx-fpm-php-8.5-release-musl"
@@ -631,13 +613,6 @@
 }
 ?>
 
-    # Compile appsec helper (C++)
-    - job: "compile appsec helper"
-      parallel:
-        matrix:
-          - ARCH: "<?= $platform['arch'] ?>"
-      artifacts: true
-
     # Compile appsec helper (Rust)
     - job: "compile appsec helper rust"
       parallel:
@@ -710,11 +685,6 @@
   needs:
     - job: "prepare code"
       artifacts: true
-    - job: "compile appsec helper"
-      parallel:
-        matrix:
-          - ARCH: "<?= $arch ?>"
-      artifacts: true
     - job: "compile appsec helper rust"
       parallel:
         matrix:

.gitlab/generate-package.php

@@ -2,15 +2,6 @@
 	path = libdatadog
 	url = https://github.com/DataDog/libdatadog
 	branch = main
-[submodule "appsec/third_party/cpp-base64"]
-	path = appsec/third_party/cpp-base64
-	url = https://github.com/ReneNyffenegger/cpp-base64
-[submodule "appsec/third_party/msgpack-c"]
-	path = appsec/third_party/msgpack-c
-	url = https://github.com/msgpack/msgpack-c.git
-[submodule "appsec/third_party/libddwaf"]
-	path = appsec/third_party/libddwaf
-	url = https://github.com/DataDog/libddwaf.git
 [submodule "tea/benchmarks/google-benchmark"]
 	path = tea/benchmarks/google-benchmark
 	url = https://github.com/google/benchmark.git

.gitmodules

@@ -148,17 +148,16 @@ delete_ini:
 	$(SUDO) rm $(INI_FILE)
 
 install_appsec:
-	cmake -S $(APPSEC_SOURCE_DIR) -B $(BUILD_DIR_APPSEC)
-	cd $(BUILD_DIR_APPSEC);make extension ddappsec-helper
-	cp $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
-	cp $(BUILD_DIR_APPSEC)/libddappsec-helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
+	cmake -S $(APPSEC_SOURCE_DIR) -DCMAKE_BUILD_TYPE=RelWithDebInfo -B $(BUILD_DIR_APPSEC)
+	$(MAKE) -C $(BUILD_DIR_APPSEC) extension
+	cp -v $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
+	cd $(APPSEC_SOURCE_DIR)/helper-rust; cargo build --release
+	cp $(APPSEC_SOURCE_DIR)/helper-rust/target/release/libddappsec_helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
 	cp $(APPSEC_SOURCE_DIR)/recommended.json /tmp/recommended.json
 	$(Q) echo "extension=ddappsec.so" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.cli_start_on_rinit=true" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.helper_path=$(PHP_EXTENSION_DIR)/libddappsec-helper.so" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.rules=/tmp/recommended.json" | $(SUDO) tee -a $(INI_FILE)
-	$(Q) echo "datadog.appsec.helper_socket_path=/tmp/ddappsec.sock" | $(SUDO) tee -a $(INI_FILE)
-	$(Q) echo "datadog.appsec.helper_lock_path=/tmp/ddappsec.lock" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.log_file=/tmp/logs/appsec.log" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.helper_log_file=/tmp/logs/helper.log" | $(SUDO) tee -a $(INI_FILE)
 

Makefile

@@ -59,7 +59,6 @@ if(DD_APPSEC_BUILD_EXTENSION)
 endif()
 
 if(DD_APPSEC_BUILD_HELPER)
-    include ("cmake/helper.cmake")
     find_program(CARGO_EXECUTABLE cargo)
     find_program(RUSTC_EXECUTABLE rustc)
     if(CARGO_EXECUTABLE AND RUSTC_EXECUTABLE)

appsec/CMakeLists.txt

@@ -74,15 +74,11 @@ cd build
 cmake ..
 make -j
 ```
-This will produce the extension, `ddappsec.so` and the helper library `libddappsec-helper.so`.
+This will produce the extension, `ddappsec.so`.
 
-Alternatively, to build the extension but not the helper, you can disable the helper build on the cmake step:
+To skip building the extension (e.g. to only run linting), you can disable it on the cmake step:
 ```
-cmake .. -DDD_APPSEC_BUILD_HELPER=OFF
-```
-Similarly, to build the helper but not the extension:
-```
-cmake .. DDD_APPSEC_BUILD_EXTENSION=OFF
+cmake .. -DDD_APPSEC_BUILD_EXTENSION=OFF
 ```
 
 #### Testing the extension
@@ -101,27 +97,11 @@ make xtest TESTS="--show-diff --show-mem -m"
 ```
 #### Testing the helper
 
-Helper tests can be located in the `dd-appsec-php/tests/helper` directory, these consist of a set of C++ unit tests written using Google Test and Mock. To build the helper tests, run the following command in the build directory:
-```
-make ddappsec_helper_test
-```
-And run the tests by executing the following command, again from the build directory:
-```
-./tests/helper/ddappsec_helper_test
-```
-To test the helper with the address and leak sanitizer, you will need to execute the cmake step with a few other options as shown below (note that it's not strictly necessary to disable the extension):
-```
-cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF \
-         -DCMAKE_CXX_FLAGS="-fsanitize=address -fsanitize=leak -DASAN_BUILD" \
-         -DCMAKE_C_FLAGS="-fsanitize=address -fsanitize=leak -DASAN_BUILD" \
-         -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-         -DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak"
-```
-After this step has concluded, build and run the helper test as before, if the sanitisers detect anything of relevance, extra output will be produced.
+The helper is implemented in Rust and lives in the `helper-rust/` directory. See `helper-rust/CLAUDE.md` for build and test instructions.
 
 ### Linting
 
-As part of our workflow, we use `clang-tidy` to lint both the extension and helper, in order to enable it add `-DDD_APPSEC_ENABLE_CLANG_TIDY=ON` to the cmake step and after building you should be able to lint by running `make tidy`.
+As part of our workflow, we use `clang-tidy` to lint the extension. To enable it, add `-DDD_APPSEC_ENABLE_CLANG_TIDY=ON` to the cmake step and after building you should be able to lint by running `make tidy`.
 
 ## Contributing