feat: complete the Kea hook's DHCP drop census and count replies (#3320)

The Kea hook's drop counter under-counts: two hard-drop paths and six
allocation/renewal refusals log without incrementing, and there is no
reply counter at all -- so grant rate and the full drop taxonomy are
both blind spots on the primary DHCP serving path. This closes both.

- `carbide_dhcp_replies_sent_total{message_type}` -- new, counts every
reply Kea actually sends (`offer`/`ack`/`nak`/`other`), incremented at
the end of `pkt4_send` and suppressed when the exchange was marked
dropped. Lease grants are the `offer`/`ack` series, matching the DPU
dhcp-server's counter of the same name.
- `carbide_dhcp_dropped_requests_total{reason}` -- the exposed name, the
`reason` label key, and every pre-existing reason string stay
byte-identical for existing dashboards, while eight previously-uncounted
paths now increment it: option-encoding failures, missing machine
context, and the six allocation/renewal refusals. A refused REQUEST
counts both its refusal and the `nak` reply that follows -- both really
happened.

Notable details:

- The raw-string reason plumbing hardens into a bounded `DropReason`
enum with a manual `LabelValue` impl pinning the legacy strings
verbatim; unrecognized FFI strings bucket to `Unknown` instead of
minting new label values.
- All events are metric-only (`log = off`): the Kea process has no
tracing subscriber, and every existing C++ LOG line stays untouched --
the C++ diff is pure one-line increments beside the drops they count.
- An option-encoding failure that throws on several options of one reply
counts that packet's drop once (status-guarded).
- The in-tree real-Kea integration tests -- the hook loaded into an
actual multi-threaded Kea -- pass with the new counters live.

Tests added!

This supports #3177

Signed-off-by: Chet Nichols III <chetn@nvidia.com>

Author: chet

Cargo.lock

@@ -30,6 +30,7 @@ tokio-util = { workspace = true }
 tonic = { workspace = true }
 
 # [local-dependencies]
+carbide-instrument = { path = "../instrument" }
 carbide-rpc = { path = "../rpc" }
 carbide-tls = { path = "../tls" }
 metrics-endpoint = { path = "../metrics-endpoint" }
@@ -40,6 +41,7 @@ dhcproto = { workspace = true }
 serde_json = { workspace = true }
 test-cdylib = { workspace = true }
 tempfile = { workspace = true }
+carbide-instrument = { path = "../instrument", features = ["test-support"] }
 carbide-rpc = { path = "../rpc" }
 carbide-test-support = { path = "../test-support" }
 prometheus = { workspace = true }

crates/dhcp/Cargo.toml

@@ -133,6 +133,11 @@ void update_option(CalloutHandle &handle, Pkt4Ptr response4_ptr,
                       "[%1]. Exception: %2")
         .arg(option)
         .arg(e.what());
+    // Several options are updated per reply and each failing update throws;
+    // count the packet's drop once, on the throw that marks it dropped.
+    if (handle.getStatus() != CalloutHandle::NEXT_STEP_DROP) {
+      carbide_increment_dropped_requests("OptionEncodingFailed");
+    }
     handle.setStatus(CalloutHandle::NEXT_STEP_DROP);
   }
 }
@@ -519,6 +524,7 @@ int pkt4_send(CalloutHandle &handle) {
     LOG_ERROR(logger, isc::log::LOG_CARBIDE_PKT4_SEND)
         .arg("Missing machine object from handle context");
     handle.setStatus(CalloutHandle::NEXT_STEP_DROP);
+    carbide_increment_dropped_requests("MissingMachineContext");
     return 1;
   }
 
@@ -543,6 +549,14 @@ int pkt4_send(CalloutHandle &handle) {
   LOG_INFO(logger, isc::log::LOG_CARBIDE_PKT4_SEND)
       .arg(response4_ptr->toText());
 
+  /*
+   * The reply is fully assembled; count it by DHCP message type unless an
+   * option failure above already marked the exchange dropped (and counted).
+   */
+  if (handle.getStatus() != CalloutHandle::NEXT_STEP_DROP) {
+    carbide_increment_reply_sent(response4_ptr->getType());
+  }
+
   return 0;
 }
 
@@ -603,6 +617,7 @@ int lease4_select(CalloutHandle &handle) {
         .arg("Missing lease4 argument");
     // At lease4_select, SKIP means Kea will not assign its selected lease.
     handle.setStatus(CalloutHandle::NEXT_STEP_SKIP);
+    carbide_increment_dropped_requests("AllocationRefused");
     return 1;
   }
 
@@ -620,6 +635,7 @@ int lease4_select(CalloutHandle &handle) {
     LOG_ERROR(logger, isc::log::LOG_CARBIDE_LEASE4_SELECT)
         .arg("Missing machine object from handle context");
     handle.setStatus(CalloutHandle::NEXT_STEP_SKIP);
+    carbide_increment_dropped_requests("AllocationRefused");
     return 1;
   }
 
@@ -631,6 +647,7 @@ int lease4_select(CalloutHandle &handle) {
     LOG_ERROR(logger, isc::log::LOG_CARBIDE_LEASE4_SELECT)
         .arg("Carbide returned no usable IPv4 address; refusing to allocate");
     handle.setStatus(CalloutHandle::NEXT_STEP_SKIP);
+    carbide_increment_dropped_requests("AllocationRefused");
     return 1;
   }
 
@@ -683,6 +700,7 @@ int lease4_renew(CalloutHandle &handle) {
         .arg("Missing lease4 argument");
     // At lease4_renew, SKIP means Kea will not update the lease database.
     handle.setStatus(CalloutHandle::NEXT_STEP_SKIP);
+    carbide_increment_dropped_requests("RenewalRefused");
     return 1;
   }
 
@@ -692,6 +710,7 @@ int lease4_renew(CalloutHandle &handle) {
     LOG_ERROR(logger, isc::log::LOG_CARBIDE_LEASE4_RENEW)
         .arg("Missing machine object from handle context");
     handle.setStatus(CalloutHandle::NEXT_STEP_SKIP);
+    carbide_increment_dropped_requests("RenewalRefused");
     return 1;
   }
 
@@ -700,6 +719,7 @@ int lease4_renew(CalloutHandle &handle) {
     LOG_ERROR(logger, isc::log::LOG_CARBIDE_LEASE4_RENEW)
         .arg("Carbide returned no usable IPv4 address; refusing to renew");
     handle.setStatus(CalloutHandle::NEXT_STEP_SKIP);
+    carbide_increment_dropped_requests("RenewalRefused");
     return 1;
   }
 

crates/dhcp/src/kea/callouts.cc

@@ -24,7 +24,6 @@ use forge_tls::default as tls_default;
 use libc::c_char;
 use metrics_endpoint::HealthController;
 use once_cell::sync::Lazy;
-use opentelemetry::metrics::Counter;
 use rpc::forge_tls_client::ForgeClientConfig;
 use tokio::runtime::{Builder, Runtime};
 
@@ -62,10 +61,11 @@ pub struct CarbideDhcpContext {
     startup_time: chrono::DateTime<chrono::Utc>,
 }
 
+// The request/drop/reply counters are `carbide-instrument` events declared in
+// `metrics.rs` and resolve from the global meter; this struct holds only the
+// state the certificate-expiry gauge reports.
 #[derive(Debug, Clone)]
 pub struct CarbideDhcpMetrics {
-    total_requests_counter: Counter<u64>,
-    dropped_requests_counter: Counter<u64>,
     forge_client_config: ForgeClientConfig,
     certificate_expiration_value: Arc<AtomicI64>,
 }
@@ -238,17 +238,37 @@ pub unsafe extern "C" fn carbide_increment_total_requests() {
     metrics::increment_total_requests();
 }
 
-/// Increments counter for number of dropped requests
+/// Increments counter for number of dropped or refused requests. The reason
+/// string is mapped onto the bounded [`metrics::DropReason`] taxonomy; a
+/// string outside the taxonomy (or a null / non-UTF-8 input) is bucketed as
+/// `Unknown` so the metric's label domain stays closed.
+///
+/// # Safety
+/// Function is unsafe as it dereferences a raw pointer given to it.  Caller is responsible
+/// to validate that the pointer passed to it meets the necessary conditions to be dereferenced.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn carbide_increment_dropped_requests(reason: *const c_char) {
+    let reason = if reason.is_null() {
+        metrics::DropReason::Unknown
+    } else {
+        unsafe { CStr::from_ptr(reason) }
+            .to_str()
+            .map_or(metrics::DropReason::Unknown, metrics::DropReason::from)
+    };
+    metrics::increment_dropped_requests(reason);
+}
+
+/// Increments counter for number of DHCP replies sent, labelled by the
+/// reply's message type. `message_type` is the raw RFC 2131 message-type code
+/// from the response packet (`Pkt4::getType()`); the mapping onto the bounded
+/// label lives in [`metrics::ReplyMessageType`].
 ///
 /// # Safety
 ///
 /// None
 #[unsafe(no_mangle)]
-pub unsafe extern "C" fn carbide_increment_dropped_requests(reason: *const c_char) {
-    unsafe {
-        let reason_value = CStr::from_ptr(reason).to_str().unwrap().to_owned();
-        metrics::increment_dropped_requests(reason_value);
-    }
+pub extern "C" fn carbide_increment_reply_sent(message_type: u8) {
+    metrics::increment_reply_sent(metrics::ReplyMessageType::from(message_type));
 }
 
 #[cfg(test)]