Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ appsec-trigger:
strategy: depend
variables:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/libddwaf appsec/third_party/libddwaf-rust

profiler-trigger:
stage: tests
Expand Down Expand Up @@ -92,6 +92,6 @@ package-trigger:
strategy: depend
variables:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/libddwaf appsec/third_party/libddwaf-rust
NIGHTLY_BUILD: $NIGHTLY_BUILD
RELIABILITY_ENV_BRANCH: $RELIABILITY_ENV_BRANCH
5 changes: 3 additions & 2 deletions .gitlab/build-appsec-helper-rust.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ cargo +nightly-"$RUST_TARGET" build \
--target "$RUST_TARGET"

# Remove musl libc dependency using patchelf (makes binary work on both musl and glibc)
BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper_rust.so"
BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper.so"
ARCH=$(uname -m)
if [ "$ARCH" = "x86_64" ]; then
patchelf --remove-needed libc.musl-x86_64.so.1 "$BINARY_PATH" 2>/dev/null || true
Expand All @@ -31,7 +31,8 @@ elif [ "$ARCH" = "aarch64" ]; then
fi

# Copy to output
cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper-rust.so"
cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper.so"
cp -v ../recommended.json "../../appsec_$(uname -m)/"

# Run tests
cargo +nightly-"$RUST_TARGET" test \
Expand Down
22 changes: 0 additions & 22 deletions .gitlab/build-appsec-helper.sh

This file was deleted.

6 changes: 2 additions & 4 deletions .gitlab/build-appsec.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,15 @@ suffix="${1:-}"
echo "Build nts extension"
switch-php "${PHP_VERSION}"
mkdir -p appsec/build ; cd appsec/build
cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_BUILD_HELPER=OFF \
-DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
make -j $MAKE_JOBS
cp -v ddappsec.so "../../appsec_$(uname -m)/ddappsec-$PHP_API${suffix}.so"
cd "../../"

echo "Build zts extension"
switch-php "${PHP_VERSION}-zts"
mkdir -p appsec/build-zts ; cd appsec/build-zts
cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_BUILD_HELPER=OFF \
-DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DDD_APPSEC_TESTING=OFF -DDD_APPSEC_EXTENSION_STATIC_LIBSTDCXX=ON
make -j $MAKE_JOBS
cp -v ddappsec.so "../../appsec_$(uname -m)/ddappsec-$PHP_API${suffix}-zts.so"
cd "../../"
Expand Down
59 changes: 3 additions & 56 deletions .gitlab/generate-appsec.php
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
- switch-php $SWITCH_PHP_VERSION
- cd appsec/build
- if [[ "$SWITCH_PHP_VERSION" == *"asan"* ]]; then ASAN_FLAG=ON; else ASAN_FLAG=OFF; fi
- "cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_HELPER=OFF
- "cmake .. -DCMAKE_BUILD_TYPE=Debug
-DCMAKE_CXX_FLAGS='-stdlib=libc++' -DCMAKE_CXX_LINK_FLAGS='-stdlib=libc++'
-DDD_APPSEC_TESTING=ON -DBOOST_CACHE_PREFIX=$CI_PROJECT_DIR/boost-cache
-DENABLE_ASAN=$ASAN_FLAG"
Expand All @@ -104,7 +104,6 @@
KUBERNETES_MEMORY_LIMIT: 30Gi
DOCKER_LOOPBACK_SIZE: 30G
ARCH: amd64
HELPER_RUST_FLAG: ""
GRADLE_USER_HOME: "$CI_PROJECT_DIR/.gradle-home"
DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED: "0"
before_script:
Expand All @@ -121,7 +120,7 @@
TERM=dumb ./gradlew loadCaches --info
fi

TERM=dumb ./gradlew $targets --info -Pbuildscan --scan $HELPER_RUST_FLAG
TERM=dumb ./gradlew $targets --info -Pbuildscan --scan
TERM=dumb ./gradlew saveCaches --info
after_script:
- mkdir -p "${CI_PROJECT_DIR}/artifacts"
Expand Down Expand Up @@ -168,19 +167,6 @@
- test8.5-release-zts
- test8.5-release-musl

"appsec integration tests (helper-rust)":
extends: .appsec_integration_tests
variables:
HELPER_RUST_FLAG: "-PuseHelperRust"
parallel:
matrix:
- targets:
- test7.4-release
- test8.1-release
- test8.3-debug
- test8.4-release-zts
- test8.5-release-musl

"helper-rust build and test":
stage: test
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
Expand Down Expand Up @@ -406,11 +392,6 @@
export PATH=$PATH:$HOME/.cargo/bin
LLVM_PROFILE_FILE="/tmp/cov-ext/%p.profraw" \
VERBOSE=1 make -j 4 xtest
- VERBOSE=1 make -j 4 ddappsec_helper_test
- |
cd ../..
LLVM_PROFILE_FILE="/tmp/cov-helper/%p.profraw" \
./appsec/build/tests/helper/ddappsec_helper_test
- |
cd /tmp/cov-ext
llvm-profdata-17 merge -sparse *.profraw -o default.profdata
Expand All @@ -420,15 +401,6 @@
echo "Uploading extension coverage to codecov"
cd "$CI_PROJECT_DIR"
codecov -t "$CODECOV_TOKEN" -n appsec-extension -v -f appsec/build/coverage-ext.lcov
- |
cd /tmp/cov-helper
llvm-profdata-17 merge -sparse *.profraw -o default.profdata
llvm-cov-17 export "$CI_PROJECT_DIR"/appsec/build/tests/helper/ddappsec_helper_test \
-format=lcov -instr-profile=default.profdata \
> "$CI_PROJECT_DIR/appsec/build/coverage-helper.lcov"
echo "Uploading helper coverage to codecov"
cd "$CI_PROJECT_DIR"
codecov -t "$CODECOV_TOKEN" -n appsec-helper -v -f appsec/build/coverage-helper.lcov
- |
echo "Uploading coverage to Datadog"
cd "$CI_PROJECT_DIR"
Expand Down Expand Up @@ -502,34 +474,9 @@
-DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" \
-DCLANG_TIDY=/usr/bin/run-clang-tidy-17 \
-DCLANG_FORMAT=/usr/bin/clang-format-17
- make -j 4 extension ddappsec-helper
- make -j 4 extension
- make format tidy

"test appsec helper asan":
stage: test
extends: .appsec_test
image: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:bookworm-6
variables:
KUBERNETES_CPU_REQUEST: 3
KUBERNETES_MEMORY_REQUEST: 3Gi
KUBERNETES_MEMORY_LIMIT: 4Gi
parallel:
matrix:
- ARCH: *arch_targets
script:
- cd appsec/build
- |
cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF \
-DDD_APPSEC_ENABLE_COVERAGE=OFF -DDD_APPSEC_TESTING=ON \
-DCMAKE_CXX_FLAGS="-stdlib=libc++ -fsanitize=address -fsanitize=leak \
-DASAN_BUILD" -DCMAKE_C_FLAGS="-fsanitize=address -fsanitize=leak \
-DASAN_BUILD" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" \
-DCLANG_TIDY=/usr/bin/run-clang-tidy-17
- make -j 4 ddappsec_helper_test
- cd ../..; ./appsec/build/tests/helper/ddappsec_helper_test

### Disabled: "we don't rely on the fuzzer these days as the protocol has been stable for a long time, so feel free to disable those jobs for now"
#"fuzz appsec helper":
# stage: test
Expand Down
30 changes: 0 additions & 30 deletions .gitlab/generate-package.php
Original file line number Diff line number Diff line change
Expand Up @@ -277,24 +277,6 @@
}
?>

"compile appsec helper":
stage: appsec
image: "registry.ddbuild.io/images/mirror/b1o7r7e0/nginx_musl_toolchain"
tags: [ "arch:$ARCH" ]
needs: [ "prepare code" ]
parallel:
matrix:
- ARCH: ["amd64", "arm64" ]
variables:
MAKE_JOBS: 12
KUBERNETES_CPU_REQUEST: 12
KUBERNETES_MEMORY_REQUEST: 4Gi
KUBERNETES_MEMORY_LIMIT: 8Gi
script: .gitlab/build-appsec-helper.sh
artifacts:
paths:
- "appsec_*"

"compile appsec helper rust":
stage: appsec
image: "registry.ddbuild.io/images/mirror/datadog/dd-appsec-php-ci:nginx-fpm-php-8.5-release-musl"
Expand Down Expand Up @@ -625,13 +607,6 @@
}
?>

# Compile appsec helper (C++)
- job: "compile appsec helper"
parallel:
matrix:
- ARCH: "<?= $platform['arch'] ?>"
artifacts: true

# Compile appsec helper (Rust)
- job: "compile appsec helper rust"
parallel:
Expand Down Expand Up @@ -704,11 +679,6 @@
needs:
- job: "prepare code"
artifacts: true
- job: "compile appsec helper"
parallel:
matrix:
- ARCH: "<?= $arch ?>"
artifacts: true
- job: "compile appsec helper rust"
parallel:
matrix:
Expand Down
6 changes: 0 additions & 6 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,6 @@
path = libdatadog
url = https://github.com/DataDog/libdatadog
branch = main
[submodule "appsec/third_party/cpp-base64"]
path = appsec/third_party/cpp-base64
url = https://github.com/ReneNyffenegger/cpp-base64
[submodule "appsec/third_party/msgpack-c"]
path = appsec/third_party/msgpack-c
url = https://github.com/msgpack/msgpack-c.git
[submodule "appsec/third_party/libddwaf"]
path = appsec/third_party/libddwaf
url = https://github.com/DataDog/libddwaf.git
Expand Down
11 changes: 5 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -148,17 +148,16 @@ delete_ini:
$(SUDO) rm $(INI_FILE)

install_appsec:
cmake -S $(APPSEC_SOURCE_DIR) -B $(BUILD_DIR_APPSEC)
cd $(BUILD_DIR_APPSEC);make extension ddappsec-helper
cp $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
cp $(BUILD_DIR_APPSEC)/libddappsec-helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
cmake -S $(APPSEC_SOURCE_DIR) -DCMAKE_BUILD_TYPE=RelWithDebInfo -B $(BUILD_DIR_APPSEC)
$(MAKE) -C $(BUILD_DIR_APPSEC) extension
cp -v $(BUILD_DIR_APPSEC)/ddappsec.so $(PHP_EXTENSION_DIR)/ddappsec.so
cd $(APPSEC_SOURCE_DIR)/helper-rust; cargo build --release
cp $(APPSEC_SOURCE_DIR)/helper-rust/target/release/libddappsec_helper.so $(PHP_EXTENSION_DIR)/libddappsec-helper.so
cp $(APPSEC_SOURCE_DIR)/recommended.json /tmp/recommended.json
$(Q) echo "extension=ddappsec.so" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.cli_start_on_rinit=true" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.helper_path=$(PHP_EXTENSION_DIR)/libddappsec-helper.so" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.rules=/tmp/recommended.json" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.helper_socket_path=/tmp/ddappsec.sock" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.helper_lock_path=/tmp/ddappsec.lock" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.log_file=/tmp/logs/appsec.log" | $(SUDO) tee -a $(INI_FILE)
$(Q) echo "datadog.appsec.helper_log_file=/tmp/logs/helper.log" | $(SUDO) tee -a $(INI_FILE)

Expand Down
5 changes: 0 additions & 5 deletions appsec/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ if(POLICY CMP0153)
cmake_policy(SET CMP0153 OLD) # allow exec_program
endif()

option(DD_APPSEC_BUILD_HELPER "Whether to builder the helper" ON)
option(DD_APPSEC_BUILD_EXTENSION "Whether to builder the extension" ON)
option(DD_APPSEC_ENABLE_COVERAGE "Whether to enable coverage calculation" OFF)
option(DD_APPSEC_TESTING "Whether to enable testing" ON)
Expand All @@ -57,10 +56,6 @@ if(DD_APPSEC_BUILD_EXTENSION)
include("cmake/extension.cmake")
endif()

if(DD_APPSEC_BUILD_HELPER)
include ("cmake/helper.cmake")
endif()

if(DD_APPSEC_DDTRACE_ALT)
include(cmake/ddtrace.cmake)
endif()
Expand Down
30 changes: 5 additions & 25 deletions appsec/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,11 @@ cd build
cmake ..
make -j
```
This will produce the extension, `ddappsec.so` and the helper library `libddappsec-helper.so`.
This will produce the extension, `ddappsec.so`.

Alternatively, to build the extension but not the helper, you can disable the helper build on the cmake step:
To skip building the extension (e.g. to only run linting), you can disable it on the cmake step:
```
cmake .. -DDD_APPSEC_BUILD_HELPER=OFF
```
Similarly, to build the helper but not the extension:
```
cmake .. DDD_APPSEC_BUILD_EXTENSION=OFF
cmake .. -DDD_APPSEC_BUILD_EXTENSION=OFF
```

#### Testing the extension
Expand All @@ -101,27 +97,11 @@ make xtest TESTS="--show-diff --show-mem -m"
```
#### Testing the helper

Helper tests can be located in the `dd-appsec-php/tests/helper` directory, these consist of a set of C++ unit tests written using Google Test and Mock. To build the helper tests, run the following command in the build directory:
```
make ddappsec_helper_test
```
And run the tests by executing the following command, again from the build directory:
```
./tests/helper/ddappsec_helper_test
```
To test the helper with the address and leak sanitizer, you will need to execute the cmake step with a few other options as shown below (note that it's not strictly necessary to disable the extension):
```
cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_BUILD_EXTENSION=OFF \
-DCMAKE_CXX_FLAGS="-fsanitize=address -fsanitize=leak -DASAN_BUILD" \
-DCMAKE_C_FLAGS="-fsanitize=address -fsanitize=leak -DASAN_BUILD" \
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak" \
-DCMAKE_MODULE_LINKER_FLAGS="-fsanitize=address -fsanitize=leak"
```
After this step has concluded, build and run the helper test as before, if the sanitisers detect anything of relevance, extra output will be produced.
The helper is implemented in Rust and lives in the `helper-rust/` directory. See `helper-rust/CLAUDE.md` for build and test instructions.

### Linting

As part of our workflow, we use `clang-tidy` to lint both the extension and helper, in order to enable it add `-DDD_APPSEC_ENABLE_CLANG_TIDY=ON` to the cmake step and after building you should be able to lint by running `make tidy`.
As part of our workflow, we use `clang-tidy` to lint the extension. To enable it, add `-DDD_APPSEC_ENABLE_CLANG_TIDY=ON` to the cmake step and after building you should be able to lint by running `make tidy`.

## Contributing

Expand Down
17 changes: 0 additions & 17 deletions appsec/cmake/check_fslib.cpp

This file was deleted.

7 changes: 1 addition & 6 deletions appsec/cmake/clang-format.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,8 @@ endif()

set(FILE_LIST "")

if(DD_APPSEC_BUILD_HELPER)
file(GLOB_RECURSE HELPER_FILES ${HELPER_SOURCE_DIR}/*.*pp tests/helper/**.cpp tests/helper/**.hpp)
list(APPEND FILE_LIST ${HELPER_FILES})
endif()

if(DD_APPSEC_BUILD_EXTENSION)
file(GLOB_RECURSE EXTENSION_FILES ${EXT_SOURCE_DIR}/*.c ${EXT_SOURCE_DIR}/*.cpp tests/helper/*.h tests/bench_helper/*.cc)
file(GLOB_RECURSE EXTENSION_FILES ${EXT_SOURCE_DIR}/*.c ${EXT_SOURCE_DIR}/*.cpp)
list(APPEND FILE_LIST ${EXTENSION_FILES})
endif()

Expand Down
4 changes: 0 additions & 4 deletions appsec/cmake/clang-tidy.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,6 @@ macro(append_target_sources target)
endforeach()
endmacro()

if(DD_APPSEC_BUILD_HELPER)
file(GLOB_RECURSE FILE_LIST ${HELPER_SOURCE_DIR}/*.*pp)
endif()

if(DD_APPSEC_BUILD_EXTENSION)
append_target_sources(extension)
endif()
Expand Down
Loading
Loading