wip move to communication via sidecar

Author: cataphract

.gitlab-ci.yml

@@ -57,7 +57,7 @@ appsec-trigger:
     strategy: depend
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
+    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/libddwaf appsec/third_party/libddwaf-rust
 
 profiler-trigger:
   stage: tests
@@ -92,6 +92,6 @@ package-trigger:
     strategy: depend
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
+    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/libddwaf appsec/third_party/libddwaf-rust
     NIGHTLY_BUILD: $NIGHTLY_BUILD
     RELIABILITY_ENV_BRANCH: $RELIABILITY_ENV_BRANCH

.gitlab/build-appsec-helper-rust.sh

@@ -22,7 +22,7 @@ cargo +nightly-"$RUST_TARGET" build \
     --target "$RUST_TARGET"
 
 # Remove musl libc dependency using patchelf (makes binary work on both musl and glibc)
-BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper_rust.so"
+BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper.so"
 ARCH=$(uname -m)
 if [ "$ARCH" = "x86_64" ]; then
     patchelf --remove-needed libc.musl-x86_64.so.1 "$BINARY_PATH" 2>/dev/null || true
@@ -31,7 +31,8 @@ elif [ "$ARCH" = "aarch64" ]; then
 fi
 
 # Copy to output
-cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper-rust.so"
+cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper.so"
+cp -v ../recommended.json "../../appsec_$(uname -m)/"
 
 # Run tests
 cargo +nightly-"$RUST_TARGET" test \

.gitlab/build-appsec-helper.sh

@@ -13,17 +13,15 @@ suffix="${1:-}"
 echo "Build nts extension"
 switch-php "${PHP_VERSION}"
 mkdir -p appsec/build ; cd appsec/build
-cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_BUILD_HELPER=OFF \
-  -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
+cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
 make -j $MAKE_JOBS
 cp -v ddappsec.so "../../appsec_$(uname -m)/ddappsec-$PHP_API${suffix}.so"
 cd "../../"
 
 echo "Build zts extension"
 switch-php "${PHP_VERSION}-zts"
 mkdir -p appsec/build-zts ; cd appsec/build-zts
-cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_BUILD_HELPER=OFF \
-  -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
+cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
 make -j $MAKE_JOBS
 cp -v ddappsec.so "../../appsec_$(uname -m)/ddappsec-$PHP_API${suffix}-zts.so"
 cd "../../"

.gitlab/build-appsec.sh

@@ -88,7 +88,7 @@
     - switch-php $SWITCH_PHP_VERSION
     - cd appsec/build
     - if [[ "$SWITCH_PHP_VERSION" == *"asan"* ]]; then ASAN_FLAG=ON; else ASAN_FLAG=OFF; fi
-    - "cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_HELPER=OFF
+    - "cmake .. -DCMAKE_BUILD_TYPE=Debug
       -DCMAKE_CXX_FLAGS='-stdlib=libc++' -DCMAKE_CXX_LINK_FLAGS='-stdlib=libc++'
 	  -DDD_APPSEC_TESTING=ON -DBOOST_CACHE_PREFIX=$CI_PROJECT_DIR/boost-cache
       -DENABLE_ASAN=$ASAN_FLAG"
@@ -104,7 +104,6 @@
     KUBERNETES_MEMORY_LIMIT: 30Gi
     DOCKER_LOOPBACK_SIZE: 30G
     ARCH: amd64
-    HELPER_RUST_FLAG: ""
     GRADLE_USER_HOME: "$CI_PROJECT_DIR/.gradle-home"
     DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED: "0"
   before_script:
@@ -121,7 +120,7 @@
         TERM=dumb ./gradlew loadCaches --info
       fi
 
-      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan $HELPER_RUST_FLAG
+      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan
       TERM=dumb ./gradlew saveCaches --info
   after_script:
     - mkdir -p "${CI_PROJECT_DIR}/artifacts"
@@ -168,19 +167,6 @@
           - test8.5-release-zts
           - test8.5-release-musl
 
-"appsec integration tests (helper-rust)":
-  extends: .appsec_integration_tests
-  variables:
-    HELPER_RUST_FLAG: "-PuseHelperRust"
-  parallel:
-    matrix:
-      - targets:
-          - test7.4-release
-          - test8.1-release
-          - test8.3-debug
-          - test8.4-release-zts
-          - test8.5-release-musl
-
 "helper-rust build and test":
   stage: test
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
@@ -406,11 +392,6 @@
       export PATH=$PATH:$HOME/.cargo/bin
       LLVM_PROFILE_FILE="/tmp/cov-ext/%p.profraw" \
         VERBOSE=1 make -j 4 xtest
-    - VERBOSE=1 make -j 4 ddappsec_helper_test
-    - |
-      cd ../..
-      LLVM_PROFILE_FILE="/tmp/cov-helper/%p.profraw" \
-        ./appsec/build/tests/helper/ddappsec_helper_test
     - |
       cd /tmp/cov-ext
       llvm-profdata-17 merge -sparse *.profraw -o default.profdata
@@ -420,15 +401,6 @@
       echo "Uploading extension coverage to codecov"
       cd "$CI_PROJECT_DIR"
       codecov -t "$CODECOV_TOKEN" -n appsec-extension -v -f appsec/build/coverage-ext.lcov
-    - |
-      cd /tmp/cov-helper
-      llvm-profdata-17 merge -sparse *.profraw -o default.profdata
-      llvm-cov-17 export "$CI_PROJECT_DIR"/appsec/build/tests/helper/ddappsec_helper_test \
-        -format=lcov -instr-profile=default.profdata \
-        > "$CI_PROJECT_DIR/appsec/build/coverage-helper.lcov"
-      echo "Uploading helper coverage to codecov"
-      cd "$CI_PROJECT_DIR"
-      codecov -t "$CODECOV_TOKEN" -n appsec-helper -v -f appsec/build/coverage-helper.lcov
     - |
       echo "Uploading coverage to Datadog"
       cd "$CI_PROJECT_DIR"
@@ -502,34 +474,9 @@
         -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" \
         -DCLANG_TIDY=/usr/bin/run-clang-tidy-17 \
         -DCLANG_FORMAT=/usr/bin/clang-format-17
-    - make -j 4 extension ddappsec-helper
+    - make -j 4 extension
     - make format tidy
 
-"test appsec helper asan":
-  stage: test
-  extends: .appsec_test
-  image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-6
-  variables:
-    KUBERNETES_CPU_REQUEST: 3
-    KUBERNETES_MEMORY_REQUEST: 3Gi
-    KUBERNETES_MEMORY_LIMIT: 4Gi
-  parallel:
-    matrix:
-      - ARCH: *arch_targets
-  script:
-    - cd appsec/build
-    - |
-      cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF \
-        -DDD_APPSEC_ENABLE_COVERAGE=OFF -DDD_APPSEC_TESTING=ON \
-        -DCMAKE_CXX_FLAGS="-stdlib=libc++ -fsanitize=address -fsanitize=leak \
-        -DASAN_BUILD" -DCMAKE_C_FLAGS="-fsanitize=address -fsanitize=leak \
-        -DASAN_BUILD" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-        -DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-        -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" \
-        -DCLANG_TIDY=/usr/bin/run-clang-tidy-17
-    - make -j 4 ddappsec_helper_test
-    - cd ../..; ./appsec/build/tests/helper/ddappsec_helper_test
-
 ### Disabled: "we don't rely on the fuzzer these days as the protocol has been stable for a long time, so feel free to disable those jobs for now"
 #"fuzz appsec helper":
 #  stage: test

.gitlab/generate-appsec.php

@@ -277,24 +277,6 @@
 }
 ?>
 
-"compile appsec helper":
-  stage: appsec
-  image: "registry.ddbuild.io/images/mirror/b1o7r7e0/nginx_musl_toolchain"
-  tags: [ "arch:$ARCH" ]
-  needs: [ "prepare code" ]
-  parallel:
-    matrix:
-      - ARCH: ["amd64", "arm64" ]
-  variables:
-    MAKE_JOBS: 12
-    KUBERNETES_CPU_REQUEST: 12
-    KUBERNETES_MEMORY_REQUEST: 4Gi
-    KUBERNETES_MEMORY_LIMIT: 8Gi
-  script: .gitlab/build-appsec-helper.sh
-  artifacts:
-    paths:
-      - "appsec_*"
-
 "compile appsec helper rust":
   stage: appsec
   image: "registry.ddbuild.io/images/mirror/datadog/dd-appsec-php-ci:nginx-fpm-php-8.5-release-musl"
@@ -625,13 +607,6 @@
 }
 ?>
 
-    # Compile appsec helper (C++)
-    - job: "compile appsec helper"
-      parallel:
-        matrix:
-          - ARCH: "<?= $platform['arch'] ?>"
-      artifacts: true
-
     # Compile appsec helper (Rust)
     - job: "compile appsec helper rust"
       parallel:
@@ -704,11 +679,6 @@
   needs:
     - job: "prepare code"
       artifacts: true
-    - job: "compile appsec helper"
-      parallel:
-        matrix:
-          - ARCH: "<?= $arch ?>"
-      artifacts: true
     - job: "compile appsec helper rust"
       parallel:
         matrix:

.gitlab/generate-package.php

@@ -2,12 +2,6 @@
 	path = libdatadog
 	url = https://github.com/DataDog/libdatadog
 	branch = main
-[submodule "appsec/third_party/cpp-base64"]
-	path = appsec/third_party/cpp-base64
-	url = https://github.com/ReneNyffenegger/cpp-base64
-[submodule "appsec/third_party/msgpack-c"]
-	path = appsec/third_party/msgpack-c
-	url = https://github.com/msgpack/msgpack-c.git
 [submodule "appsec/third_party/libddwaf"]
 	path = appsec/third_party/libddwaf
 	url = https://github.com/DataDog/libddwaf.git

.gitmodules

@@ -148,17 +148,16 @@ delete_ini:
 	$(SUDO) rm $(INI_FILE)
 
 install_appsec:
-	cmake -S $(APPSEC_SOURCE_DIR) -B $(BUILD_DIR_APPSEC)
-	cd $(BUILD_DIR_APPSEC);make extension ddappsec-helper
-	cp $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
-	cp $(BUILD_DIR_APPSEC)/libddappsec-helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
+	cmake -S $(APPSEC_SOURCE_DIR) -DCMAKE_BUILD_TYPE=RelWithDebInfo -B $(BUILD_DIR_APPSEC)
+	$(MAKE) -C $(BUILD_DIR_APPSEC) extension
+	cp -v $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
+	cd $(APPSEC_SOURCE_DIR)/helper-rust; cargo build --release
+	cp $(APPSEC_SOURCE_DIR)/helper-rust/target/release/libddappsec_helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
 	cp $(APPSEC_SOURCE_DIR)/recommended.json /tmp/recommended.json
 	$(Q) echo "extension=ddappsec.so" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.cli_start_on_rinit=true" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.helper_path=$(PHP_EXTENSION_DIR)/libddappsec-helper.so" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.rules=/tmp/recommended.json" | $(SUDO) tee -a $(INI_FILE)
-	$(Q) echo "datadog.appsec.helper_socket_path=/tmp/ddappsec.sock" | $(SUDO) tee -a $(INI_FILE)
-	$(Q) echo "datadog.appsec.helper_lock_path=/tmp/ddappsec.lock" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.log_file=/tmp/logs/appsec.log" | $(SUDO) tee -a $(INI_FILE)
 	$(Q) echo "datadog.appsec.helper_log_file=/tmp/logs/helper.log" | $(SUDO) tee -a $(INI_FILE)
 

Makefile

@@ -32,7 +32,6 @@ if(POLICY CMP0153)
     cmake_policy(SET CMP0153 OLD) # allow exec_program
 endif()
 
-option(DD_APPSEC_BUILD_HELPER "Whether to builder the helper" ON)
 option(DD_APPSEC_BUILD_EXTENSION "Whether to builder the extension" ON)
 option(DD_APPSEC_ENABLE_COVERAGE "Whether to enable coverage calculation" OFF)
 option(DD_APPSEC_TESTING "Whether to enable testing" ON)
@@ -57,10 +56,6 @@ if(DD_APPSEC_BUILD_EXTENSION)
     include("cmake/extension.cmake")
 endif()
 
-if(DD_APPSEC_BUILD_HELPER)
-    include ("cmake/helper.cmake")
-endif()
-
 if(DD_APPSEC_DDTRACE_ALT)
     include(cmake/ddtrace.cmake)
 endif()

appsec/CMakeLists.txt

@@ -74,15 +74,11 @@ cd build
 cmake ..
 make -j
 ```
-This will produce the extension, `ddappsec.so` and the helper library `libddappsec-helper.so`.
+This will produce the extension, `ddappsec.so`.
 
-Alternatively, to build the extension but not the helper, you can disable the helper build on the cmake step:
+To skip building the extension (e.g. to only run linting), you can disable it on the cmake step:
 ```
-cmake .. -DDD_APPSEC_BUILD_HELPER=OFF
-```
-Similarly, to build the helper but not the extension:
-```
-cmake .. DDD_APPSEC_BUILD_EXTENSION=OFF
+cmake .. -DDD_APPSEC_BUILD_EXTENSION=OFF
 ```
 
 #### Testing the extension
@@ -101,27 +97,11 @@ make xtest TESTS="--show-diff --show-mem -m"
 ```
 #### Testing the helper
 
-Helper tests can be located in the `dd-appsec-php/tests/helper` directory, these consist of a set of C++ unit tests written using Google Test and Mock. To build the helper tests, run the following command in the build directory:
-```
-make ddappsec_helper_test
-```
-And run the tests by executing the following command, again from the build directory:
-```
-./tests/helper/ddappsec_helper_test
-```
-To test the helper with the address and leak sanitizer, you will need to execute the cmake step with a few other options as shown below (note that it's not strictly necessary to disable the extension):
-```
-cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF \
-         -DCMAKE_CXX_FLAGS="-fsanitize=address -fsanitize=leak -DASAN_BUILD" \
-         -DCMAKE_C_FLAGS="-fsanitize=address -fsanitize=leak -DASAN_BUILD" \
-         -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-         -DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak"
-```
-After this step has concluded, build and run the helper test as before, if the sanitisers detect anything of relevance, extra output will be produced.
+The helper is implemented in Rust and lives in the `helper-rust/` directory. See `helper-rust/CLAUDE.md` for build and test instructions.
 
 ### Linting
 
-As part of our workflow, we use `clang-tidy` to lint both the extension and helper, in order to enable it add `-DDD_APPSEC_ENABLE_CLANG_TIDY=ON` to the cmake step and after building you should be able to lint by running `make tidy`.
+As part of our workflow, we use `clang-tidy` to lint the extension. To enable it, add `-DDD_APPSEC_ENABLE_CLANG_TIDY=ON` to the cmake step and after building you should be able to lint by running `make tidy`.
 
 ## Contributing