feat(deployment): add sd_notify integration with watchdog support

Cargo.toml

@@ -461,6 +461,7 @@ tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.18.0", default-features = false }
+sd-notify = { version = "0.5.0", default-features = false }
 
 [build-dependencies]
 prost-build = { workspace = true, optional = true }

LICENSE-3rdparty.csv

@@ -668,6 +668,7 @@ schemars,https://github.com/GREsau/schemars,MIT,Graham Esau <gesau@hotmail.co.uk
 scoped-tls,https://github.com/alexcrichton/scoped-tls,MIT OR Apache-2.0,Alex Crichton <alex@alexcrichton.com>
 scopeguard,https://github.com/bluss/scopeguard,MIT OR Apache-2.0,bluss
 sct,https://github.com/rustls/sct.rs,Apache-2.0 OR ISC OR MIT,Joseph Birr-Pixton <jpixton@gmail.com>
+sd-notify,https://github.com/lnicola/sd-notify,MIT OR Apache-2.0,The sd-notify Authors
 seahash,https://gitlab.redox-os.org/redox-os/seahash,MIT,"ticki <ticki@users.noreply.github.com>, Tom Almeida <tom@tommoa.me>"
 sec1,https://github.com/RustCrypto/formats/tree/master/sec1,Apache-2.0 OR MIT,RustCrypto Developers
 secrecy,https://github.com/iqlusioninc/crates/tree/main/secrecy,Apache-2.0 OR MIT,Tony Arcieri <tony@iqlusion.io>

changelog.d/systemd_notify.feature.md

@@ -0,0 +1,4 @@
+Add systemd notify integration. Vector now sends `READY=1` when fully started, `STOPPING=1`
+when beginning a graceful shutdown, and `WATCHDOG=1` pings at half the configured `WatchdogSec`
+interval. The bundled `vector.service` and `hardened-vector.service` unit files are updated
+to use `Type=notify`, with an optional `WatchdogSec` directive.

distribution/systemd/hardened-vector.service

@@ -11,6 +11,7 @@ After=network-online.target
 Requires=network-online.target
 
 [Service]
+Type=notify
 EnvironmentFile=-/etc/default/vector
 User=vector
 Group=vector
@@ -20,6 +21,9 @@ ExecReload=/usr/bin/vector validate
 ExecReload=/bin/kill -HUP $MAINPID
 Restart=no
 
+# Uncomment to enable watchdog. Vector will ping at half this interval.
+#WatchdogSec=30
+
 # capabilities
 AmbientCapabilities=CAP_NET_BIND_SERVICE
 CapabilityBoundingSet=CAP_NET_BIND_SERVICE

distribution/systemd/vector.service

@@ -5,6 +5,7 @@ After=network-online.target
 Requires=network-online.target
 
 [Service]
+Type=notify
 User=vector
 Group=vector
 ExecStartPre=/usr/bin/vector validate
@@ -18,5 +19,9 @@ EnvironmentFile=-/etc/default/vector
 # it is also supported to have it here.
 StartLimitInterval=10
 StartLimitBurst=5
+
+# Uncomment to enable watchdog. Vector will ping at half this interval.
+#WatchdogSec=30
+
 [Install]
 WantedBy=multi-user.target

src/app.rs

@@ -257,6 +257,8 @@ impl Application {
 
         emit!(VectorStarted);
         handle.spawn(heartbeat::heartbeat());
+        #[cfg(target_os = "linux")]
+        handle.spawn(crate::systemd::watchdog());
 
         let Self {
             root_opts,
@@ -276,6 +278,9 @@ impl Application {
             extra_context: config.extra_context,
         });
 
+        #[cfg(target_os = "linux")]
+        crate::systemd::sd_notify_ready();
+
         Ok(StartedApplication {
             config_paths: config.config_paths,
             internal_topologies: config.internal_topologies,
@@ -493,6 +498,8 @@ impl FinishedApplication {
     }
 
     async fn stop(topology_controller: TopologyController, mut signal_rx: SignalRx) -> ExitStatus {
+        #[cfg(target_os = "linux")]
+        crate::systemd::sd_notify_stopping();
         emit!(VectorStopped);
         tokio::select! {
             _ = topology_controller.stop() => ExitStatus::from_raw({

src/lib.rs

@@ -112,6 +112,8 @@ pub(crate) mod sink_ext;
 pub mod sinks;
 #[allow(unreachable_pub)]
 pub mod sources;
+#[cfg(target_os = "linux")]
+pub mod systemd;
 #[cfg(feature = "api-client")]
 #[allow(unreachable_pub)]
 pub mod tap;

src/systemd.rs

@@ -0,0 +1,48 @@
+//! Systemd integration via `sd_notify`
+//! See <https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html>
+
+use tokio::time::interval;
+
+/// Sends `READY=1` to systemd via sd_notify. No-op if not Type=notify.
+pub fn sd_notify_ready() {
+    if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) {
+        warn!(message = "Failed to notify systemd of ready state.", %error);
+    }
+}
+
+/// Sends `STOPPING=1` to systemd via sd_notify. No-op if not Type=notify.
+pub fn sd_notify_stopping() {
+    if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Stopping]) {
+        warn!(message = "Failed to notify systemd of stopping state.", %error);
+    }
+}
+
+/// Sends `WATCHDOG=1` to systemd via sd_notify. No-op if not Type=notify.
+pub fn sd_notify_watchdog() {
+    if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) {
+        warn!(message = "Failed to send systemd watchdog ping.", %error);
+    }
+}
+
+/// Sends `WATCHDOG=1` pings at half the `WatchdogSec` interval. No-op if not set.
+pub async fn watchdog() {
+    let Some(duration) = sd_notify::watchdog_enabled() else {
+        return;
+    };
+    let mut ticker = interval(duration / 2);
+    loop {
+        ticker.tick().await;
+        sd_notify_watchdog();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn sd_notify_no_socket_does_not_panic() {
+        // NOTIFY_SOCKET is not set in test environments - these must be no-ops.
+        super::sd_notify_ready();
+        super::sd_notify_stopping();
+        super::sd_notify_watchdog();
+    }
+}