Adds process tags to remote config payload (#3658)

* bump libdatadog

* Adds process tags to remote config payload

* now process tags are passed as a vec

* fix appsec test

* leiyks review

* bump libdatadog

* fix build

* Fix build

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Fix build

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

* Fix permission issue in appsec int build.gradle

* Pin new version of image nginx-fpm-php-8.5-release-musl

---------

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>
Co-authored-by: Bob Weinand <bob.weinand@datadoghq.com>
Co-authored-by: Gustavo André dos Santos Lopes <gustavo.lopes@datadoghq.com>

Author: 3 people

Cargo.lock

@@ -48,7 +48,7 @@ RUN_TESTS_CMD := DD_SERVICE= DD_ENV= REPORT_EXIT_STATUS=1 TEST_PHP_SRCDIR=$(PROJ
 
 C_FILES = $(shell find components components-rs ext src/dogstatsd zend_abstract_interface -name '*.c' -o -name '*.h' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
 TEST_FILES = $(shell find tests/ext -name '*.php*' -o -name '*.inc' -o -name '*.json' -o -name '*.yaml' -o -name 'CONFLICTS' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
-RUST_FILES = $(BUILD_DIR)/Cargo.toml $(BUILD_DIR)/Cargo.lock $(shell find components-rs -name '*.c' -o -name '*.rs' -o -name 'Cargo.toml' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(shell find libdatadog/{build-common,datadog-ffe,datadog-ipc,datadog-ipc-macros,datadog-live-debugger,datadog-live-debugger-ffi,datadog-remote-config,datadog-sidecar,datadog-sidecar-ffi,datadog-sidecar-macros,libdd-alloc,libdd-common,libdd-common-ffi,libdd-crashtracker,libdd-crashtracker-ffi,libdd-data-pipeline,libdd-ddsketch,libdd-dogstatsd-client,libdd-library-config,libdd-library-config-ffi,libdd-log,libdd-telemetry,libdd-telemetry-ffi,libdd-tinybytes,libdd-trace-*,spawn_worker,tools/{cc_utils,sidecar_mockgen},libdd-trace-*,Cargo.toml} \( -type l -o -type f \) \( -path "*/src*" -o -path "*/examples*" -o -path "*Cargo.toml" -o -path "*/build.rs" -o -path "*/tests/dataservice.rs" -o -path "*/tests/service_functional.rs" \) -not -path "*/datadog-ipc/build.rs" -not -path "*/datadog-sidecar-ffi/build.rs")
+RUST_FILES = $(BUILD_DIR)/Cargo.toml $(BUILD_DIR)/Cargo.lock $(shell find components-rs -name '*.c' -o -name '*.rs' -o -name 'Cargo.toml' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(shell find libdatadog/{build-common,datadog-ffe,datadog-ipc,datadog-ipc-macros,datadog-live-debugger,datadog-live-debugger-ffi,datadog-remote-config,datadog-sidecar,datadog-sidecar-ffi,datadog-sidecar-macros,libdd-alloc,libdd-common,libdd-common-ffi,libdd-crashtracker,libdd-crashtracker-ffi,libdd-data-pipeline,libdd-ddsketch,libdd-dogstatsd-client,libdd-library-config,libdd-library-config-ffi,libdd-log,libdd-libunwind-sys,libdd-telemetry,libdd-telemetry-ffi,libdd-tinybytes,libdd-trace-*,spawn_worker,tools/{cc_utils,sidecar_mockgen},libdd-trace-*,Cargo.toml} \( -type l -o -type f \) \( -path "*/src*" -o -path "*/examples*" -o -path "*/libunwind*" -o -path "*Cargo.toml" -o -path "*/build.rs" -o -path "*/tests/dataservice.rs" -o -path "*/tests/service_functional.rs" \) -not -path "*/datadog-ipc/build.rs" -not -path "*/datadog-sidecar-ffi/build.rs")
 ALL_OBJECT_FILES = $(C_FILES) $(RUST_FILES) $(BUILD_DIR)/Makefile
 TEST_OPCACHE_FILES = $(shell find tests/opcache -name '*.php*' -o -name '.gitkeep' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
 TEST_STUB_FILES = $(shell find tests/ext -type d -name 'stubs' -exec find '{}' -type f \; | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
@@ -500,7 +500,9 @@ $(PACKAGES_BUILD_DIR)/datadog-setup.php: $(PACKAGES_BUILD_DIR)
 		$(PACKAGES_BUILD_DIR)
 
 build_pecl_package:
-	echo $(subst $(BUILD_DIR)/,,$(C_FILES) $(RUST_FILES) $(TEST_FILES) $(TEST_STUB_FILES) $(M4_FILES) Cargo.lock) | tooling/bin/pecl-build
+	$(file >.pecl-files,$(subst $(BUILD_DIR)/,,$(C_FILES) $(RUST_FILES) $(TEST_FILES) $(TEST_STUB_FILES) $(M4_FILES) Cargo.lock))
+	tooling/bin/pecl-build < .pecl-files
+	@rm -f .pecl-files
 
 dbgsym.tar.gz: $(PACKAGES_BUILD_DIR)
 	$(if $(DDTRACE_MAKE_PACKAGES_ASAN), , tar -zcf $(PACKAGES_BUILD_DIR)/dd-library-php-$(VERSION)_windows_debugsymbols.tar.gz ./extensions_x86_64_debugsymbols --owner=0 --group=0)

Makefile

@@ -189,10 +189,10 @@ def buildRunInDockerTask = { Map options ->
     }
     if (options.get('needsBoostCache', true)) {
         if (project.hasProperty('localBoostCache')) {
-            binds[project.getProperty('localBoostCache')] = '/root/.boost'
+            binds[project.getProperty('localBoostCache')] = '/var/boost-cache'
         } else {
             volumes['php-appsec-boost-cache'] = [
-                    mountPoint: '/root/.boost',
+                    mountPoint: '/var/boost-cache',
                     task: createVolumeTask('php-appsec-boost-cache'),
             ]
         }
@@ -390,7 +390,7 @@ def buildAppSecTask = { String version, String variant, altBaseTag = null ->
                               -DDD_APPSEC_ENABLE_PATCHELF_LIBC=ON \\
                               -DGIT_COMMIT=${libddwafCommit()} \\
                               -DDD_APPSEC_TESTING=ON \\
-                              -DBOOST_CACHE_PREFIX=/root/.boost /project/appsec
+                              -DBOOST_CACHE_PREFIX=/var/boost-cache /project/appsec
                     make -j extension ddappsec-helper && \\
                         touch ddappsec.so libddappsec-helper.so
                 """

appsec/tests/integration/build.gradle

@@ -137,5 +137,5 @@ ext.tag_mappings = [
     'nginx-fpm-php-8.5-release-zts': 'sha256:527f2ba273685ff7e3f72a74076fd6d99cbc87d51fb3485fef4dc80a0cfddeee',
     'nginx-fpm-php-8.5-release': 'sha256:fc2eea1ed06dfcf8102bdbfb2596f9a32bbe6493ca83d53370e1438289caa6aa',
     'nginx-fpm-php-8.5-debug': 'sha256:915d0155897e4f42cf1867b93edbb928afcc1c5b3bd3eccdb2d56ac50510cd82',
-	'nginx-fpm-php-8.5-release-musl': 'sha256:73857002765b217e1076adc04f9ebc88505b0fa77fc48e2ad3005594b2f39bc9',
+	'nginx-fpm-php-8.5-release-musl': 'sha256:a1ef09c1bf398049b340ab2a1c40839d828420df5a404831bb7b33a09d433755',
 ]

appsec/tests/integration/gradle/tag_mappings.gradle

@@ -3,9 +3,9 @@ FROM php:8.5-fpm-alpine
 # doubles as php/toolchain image for musl
 
 RUN apk add --no-cache \
-    autoconf bash bison clang clang-dev cmake curl curl-dev \
+    autoconf automake bash bison clang clang-dev cmake curl curl-dev \
     g++ gcc gdb git libc-dev linux-headers llvm-libunwind-static make \
-    musl-dev nginx oniguruma-dev openssl-dev patchelf pkgconf re2c \
+    musl-dev nginx oniguruma-dev openssl-dev patchelf pkgconf re2c libtool \
     libxml2-dev libzip-dev xz-static zlib-dev \
     vim
 

appsec/tests/integration/src/docker/nginx-fpm-musl/Dockerfile

@@ -218,6 +218,12 @@ public static class TracerInfo {
             @JsonProperty("app_version")
             public String serviceVersion;
 
+            @JsonProperty("process_tags")
+            public List<String> processTags;
+
+            @JsonProperty("container_tags")
+            public List<String> containerTags;
+
             @Override
             public String toString() {
                 return new StringJoiner(", ", TracerInfo.class.getSimpleName() + "[", "]")
@@ -229,6 +235,8 @@ public String toString() {
                         .add("extraServices=" + extraServices)
                         .add("serviceEnv='" + serviceEnv + "'")
                         .add("serviceVersion='" + serviceVersion + "'")
+                        .add("processTags=" + processTags)
+                        .add("container_tags=" + containerTags)
                         .toString();
             }
         }

appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/rem_cfg/RemoteConfigRequest.java

@@ -23,6 +23,7 @@ rename_types = "PascalCase"
 "PushTagResult" = "ddog_Vec_Tag_PushResult"
 "ZendString" = "_zend_string"
 "FILE" = "FILE"
+"EndpointConfig" = "ddog_crasht_EndpointConfig"
 
 [enum]
 prefix_with_name = true

cbindgen.toml

@@ -39,6 +39,12 @@
 #  define DDOG_CHECK_RETURN
 #endif
 
+#ifdef _WIN32
+#define LIBDD_DLLIMPORT __declspec(dllimport)
+#else
+#define LIBDD_DLLIMPORT
+#endif
+
 /**
  * Default value for the timeout field in milliseconds.
  */
@@ -1231,9 +1237,6 @@ typedef enum ddog_crasht_SignalNames {
  * variable to allow downgrading the collector.
  */
 typedef enum ddog_crasht_StacktraceCollection {
-  /**
-   * Stacktrace collection occurs in the
-   */
   DDOG_CRASHT_STACKTRACE_COLLECTION_DISABLED,
   DDOG_CRASHT_STACKTRACE_COLLECTION_WITHOUT_SYMBOLS,
   /**
@@ -1286,6 +1289,14 @@ typedef struct ddog_crasht_Slice_CharSlice {
   uintptr_t len;
 } ddog_crasht_Slice_CharSlice;
 
+typedef struct ddog_crasht_EndpointConfig {
+  ddog_CharSlice url;
+  ddog_CharSlice api_key;
+  ddog_CharSlice test_token;
+  uint64_t timeout;
+  bool use_system_resolver;
+} ddog_crasht_EndpointConfig;
+
 typedef struct ddog_crasht_Slice_I32 {
   /**
    * Should be non-null and suitably aligned for the underlying type. It is
@@ -1308,7 +1319,7 @@ typedef struct ddog_crasht_Config {
    * The endpoint to send the crash report to (can be a file://).
    * If None, the crashtracker will infer the agent host from env variables.
    */
-  const struct ddog_Endpoint *endpoint;
+  struct ddog_crasht_EndpointConfig endpoint;
   /**
    * Optional filename for a unix domain socket if the receiver is used asynchonously
    */
@@ -1388,6 +1399,14 @@ typedef struct ddog_crasht_Slice_CInt {
   uintptr_t len;
 } ddog_crasht_Slice_CInt;
 
+/**
+ * Represents an object that should only be referred to by its handle.
+ * Do not access its member for any reason, only use the C API functions on this struct.
+ */
+typedef struct ddog_crasht_Handle_StackTrace {
+  struct ddog_crasht_StackTrace *inner;
+} ddog_crasht_Handle_StackTrace;
+
 /**
  * A generic result type for when an operation may fail,
  * or may return <T> in case of success.
@@ -1493,14 +1512,6 @@ typedef struct ddog_crasht_Span {
   ddog_CharSlice thread_name;
 } ddog_crasht_Span;
 
-/**
- * Represents an object that should only be referred to by its handle.
- * Do not access its member for any reason, only use the C API functions on this struct.
- */
-typedef struct ddog_crasht_Handle_StackTrace {
-  struct ddog_crasht_StackTrace *inner;
-} ddog_crasht_Handle_StackTrace;
-
 typedef struct ddog_crasht_ThreadData {
   bool crashed;
   ddog_CharSlice name;
@@ -1875,6 +1886,13 @@ void ddog_endpoint_set_timeout(struct ddog_Endpoint *endpoint, uint64_t millis);
 
 void ddog_endpoint_set_test_token(struct ddog_Endpoint *endpoint, ddog_CharSlice token);
 
+/**
+ * Set whether to use the system DNS resolver when building the reqwest client.
+ * If false, the default in-process resolver is used.
+ */
+void ddog_endpoint_set_use_system_resolver(struct ddog_Endpoint *endpoint,
+                                           bool use_system_resolver);
+
 void ddog_endpoint_drop(struct ddog_Endpoint*);
 
 struct ddog_Option_U32 ddog_Option_U32_some(uint32_t v);

components-rs/common.h

@@ -131,6 +131,45 @@ struct ddog_VoidResult ddog_crasht_init_without_receiver(struct ddog_crasht_Conf
  */
 struct ddog_crasht_Slice_CInt ddog_crasht_default_signals(void);
 
+/**
+ * Report an unhandled exception as a crash event.
+ *
+ * This function sends a crash report for an unhandled exception detected
+ * by the runtime. It is intended to be called when the process is in a
+ * terminal state due to an unhandled exception.
+ *
+ * # Parameters
+ * - `error_type`: Optional type/class of the exception (e.g. "NullPointerException"). Pass empty
+ *   CharSlice for unknown.
+ * - `error_message`: Optional error message. Pass empty CharSlice for no message.
+ * - `runtime_stack`: Stack trace from the runtime. Consumed by this call.
+ *
+ * If the crash-tracker has not been initialized, this function is a no-op.
+ *
+ * # Side effects
+ *   This function disables the signal-based crash handler before performing
+ *   any work. This means that if the process receives a fatal signal (SIGSEGV)
+ *   during or after this call, the crashtracker will not produce a
+ *   second crash report. The previous signal handler (if any) will still be
+ *   chained.
+ *
+ * # Failure mode
+ *   If a fatal signal occurs while this function is in progress, the calling
+ *   process is in an unrecoverable state; the crashtracker cannot report the
+ *   secondary fault and the caller's own signal handler (if any) will execute
+ *   in a potentially corrupted context. Callers should treat this function as a
+ *   terminal operation and exit shortly after it returns.
+ *
+ * # Safety
+ *   Crash-tracking functions are not reentrant.
+ *   No other crash-handler functions should be called concurrently.
+ *   The `runtime_stack` handle must be valid and will be consumed.
+ */
+DDOG_CHECK_RETURN
+struct ddog_VoidResult ddog_crasht_report_unhandled_exception(ddog_CharSlice error_type,
+                                                              ddog_CharSlice error_message,
+                                                              struct ddog_crasht_Handle_StackTrace *runtime_stack);
+
 /**
  * Removes all existing additional tags
  * Expected to be used after a fork, to reset the additional tags on the child
@@ -661,7 +700,12 @@ struct ddog_VoidResult ddog_crasht_CrashInfoBuilder_with_thread_name(struct ddog
  * The `builder` can be null, but if non-null it must point to a Builder made by this module,
  * which has not previously been dropped.
  * All arguments must be valid.
- * This method requires that the builder has a UUID and metadata set
+ * This method requires that the builder has `metadata` and `kind` set
+ * Applications can add `message` or `sig_info` to the builder to provide additional context.
+ * If set, the data will be used to derive the crash ping message in the order of
+ * - an explicit message set with `with_message`
+ * - sig_info set with `with_sig_info`
+ * - kind set with `with_kind`
  */
 DDOG_CHECK_RETURN
 struct ddog_VoidResult ddog_crasht_CrashInfoBuilder_upload_ping_to_endpoint(struct ddog_crasht_Handle_CrashInfoBuilder *builder,

components-rs/crashtracker.h

@@ -39,6 +39,10 @@ void ddtrace_drop_rust_string(char *input, uintptr_t len);
 
 struct ddog_Endpoint *ddtrace_parse_agent_url(ddog_CharSlice url);
 
+void ddtrace_endpoint_as_crashtracker_config(const struct ddog_Endpoint *endpoint,
+                                             void (*callback)(ddog_crasht_EndpointConfig, void*),
+                                             void *userdata);
+
 ddog_Configurator *ddog_library_configurator_new_dummy(bool debug_logs, ddog_CharSlice language);
 
 int posix_spawn_file_actions_addchdir_np(void *file_actions, const char *path);
@@ -80,7 +84,8 @@ bool ddog_remote_configs_service_env_change(struct ddog_RemoteConfigState *remot
                                             ddog_CharSlice service,
                                             ddog_CharSlice env,
                                             ddog_CharSlice version,
-                                            const struct ddog_Vec_Tag *tags);
+                                            const struct ddog_Vec_Tag *tags,
+                                            const struct ddog_Vec_Tag *process_tags);
 
 bool ddog_remote_config_alter_dynamic_config(struct ddog_RemoteConfigState *remote_config,
                                              ddog_CharSlice config,