feat: count firmware update outcomes across every target

Firmware updates run for three targets -- host BIOS/BMC, DPU NICs, and SuperNICs -- and none of their outcomes are countable today: starts, completions, and failures live only in log lines, including the "wrong version after attempted update" case that means an update silently didn't take. This makes all of it countable, with each event replacing its site's log line at the historical level (info for progress, warn for failures) and the same information as structured fields.

Two counters from `carbide-instrument`:

- `carbide_firmware_updates_total{target, phase}` -- started and completed, per target. For hosts both phases fire, so the started-to-completed gap is the in-flight backlog; the DPU-NIC and SuperNIC targets emit started only, with completion visible in their own state flows.
- `carbide_firmware_update_failures_total{target, cause}` -- `no_update_match` (a DPU left ready between snapshot and trigger) and `wrong_version_after_update` (the update ran and the firmware version still doesn't match -- the silent-failure case).

Notable details:

- Started counts only committed triggers: the DPU-NIC emit fires after `txn.commit()`, so an update whose trigger rolled back is a `no_update_match` failure, never a started update, and the started-minus-completed math stays truthful.
- Host completion emits once per machine as its update markers clear, replacing one Vec-debug log line with per-machine lines.
- A SuperNIC re-counts one started per scout pass while a device stays mismatched -- each pass issues a real apply command, and the sustained rate is the stuck-device signal.

Tests added!

This supports #3175

Signed-off-by: Chet Nichols III <chetn@nvidia.com>

Author: chet

crates/api-core/src/handlers/host_reprovisioning.rs

@@ -79,6 +79,17 @@ pub(crate) async fn trigger_host_reprovisioning(
                 &machine_id,
             )
             .await?;
+            // Manual initiations pair with the same completion emit the
+            // update manager's automatic path gets, keeping the
+            // started-to-completed gap truthful for every initiator.
+            carbide_instrument::emit(
+                crate::machine_update_manager::metrics::FirmwareUpdateProgress {
+                    target: crate::machine_update_manager::metrics::FirmwareUpdateTarget::Host,
+                    phase: crate::machine_update_manager::metrics::FirmwareUpdatePhase::Started,
+                    machine_id,
+                    detail: initiator.to_string(),
+                },
+            );
         }
         Mode::Clear => {
             db::host_machine_update::clear_host_reprovisioning_request(&mut txn, &machine_id)

crates/api-core/src/handlers/svpc.rs

@@ -34,6 +34,9 @@ use rpc::protos::mlx_device::MlxDeviceInfo;
 use tonic::{Request, Response, Status};
 
 use crate::api::{Api, log_request_data};
+use crate::machine_update_manager::metrics::{
+    FirmwareUpdatePhase, FirmwareUpdateProgress, FirmwareUpdateTarget,
+};
 use crate::{CarbideError, CarbideResult};
 
 // Code to handle SVPC specific information.
@@ -197,12 +200,16 @@ fn build_apply_firmware_command<'a>(
             return None;
         }
 
-        tracing::info!(
-            %machine_id, %pci_name, %part_number, %psid,
-            observed_fw_version = ?device_info.fw_version_current,
-            expected_fw_version = %fw_profile.firmware_spec.version,
-            "firmware version mismatch, applying firmware"
-        );
+        carbide_instrument::emit(FirmwareUpdateProgress {
+            target: FirmwareUpdateTarget::SuperNic,
+            phase: FirmwareUpdatePhase::Started,
+            machine_id,
+            detail: format!(
+                "pci_name={pci_name} part_number={part_number} psid={psid} \
+                 observed_fw_version={:?} expected_fw_version={}",
+                device_info.fw_version_current, fw_profile.firmware_spec.version
+            ),
+        });
         Some(Cow::Borrowed(fw_profile))
     })();
 

crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs

@@ -29,6 +29,10 @@ use sqlx::PgConnection;
 
 use super::dpu_nic_firmware_metrics::DpuNicFirmwareUpdateMetrics;
 use super::machine_update_module::MachineUpdateModule;
+use super::metrics::{
+    FirmwareUpdateFailed, FirmwareUpdateFailureCause, FirmwareUpdatePhase, FirmwareUpdateProgress,
+    FirmwareUpdateTarget,
+};
 use crate::cfg::file::CarbideConfig;
 use crate::machine_update_manager::MachineUpdateManager;
 use crate::{CarbideResult, DatabaseError};
@@ -46,6 +50,14 @@ pub struct DpuNicFirmwareUpdate {
     /// DPF handle for discovering outdated DPF-managed DPUs. `None` when DPF
     /// is disabled in config; in that case `find_outdated_dpus_dpf` is not called.
     pub dpf: Option<Arc<dyn DpfOperations>>,
+    /// Wrong-version outcomes already counted, keyed by DPU and the version it
+    /// landed on: the condition persists across manager passes (the update
+    /// marker stays until an operator intervenes or a new update runs), and a
+    /// counter must record the outcome once, not once per poll. An entry
+    /// clears when the DPU's markers clear, so a later attempt that lands
+    /// wrong again is a new outcome. In-memory: a restart re-reports at most
+    /// once per stuck DPU.
+    pub reported_wrong_versions: std::sync::Mutex<HashSet<(MachineId, String)>>,
 }
 
 #[async_trait]
@@ -105,11 +117,6 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
                 output + format!("{} ({}) ", dpu.dpu_machine_id, dpu.firmware_version).as_str()
             });
 
-            tracing::info!(
-                "Starting DPU updates for host {}: {}",
-                host_machine_id,
-                dpu_update_string
-            );
             // If the reprovisioning failed to update the database for a
             // given {dpu,host}_machine_id, log it as a warning and don't
             // add it to updates_started.
@@ -124,11 +131,13 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
             {
                 match reprovisioning_err {
                     DatabaseError::NotFoundError { id, .. } => {
-                        tracing::warn!(
-                            "failed to trigger reprovisioning for managed host : {} - no update match for id: {}",
-                            host_machine_id,
-                            id
-                        );
+                        carbide_instrument::emit(FirmwareUpdateFailed {
+                            target: FirmwareUpdateTarget::DpuNic,
+                            cause: FirmwareUpdateFailureCause::NoUpdateMatch,
+                            machine_id: host_machine_id,
+                            unmatched_dpu_machine_id: id,
+                            firmware_version: String::new(),
+                        });
                         continue;
                     }
                     _ => {
@@ -139,6 +148,15 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
 
             txn.commit().await?;
 
+            // Counted only once the trigger is committed: a DPU that left
+            // ready between snapshot and trigger is a NoUpdateMatch failure,
+            // not a started update.
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::DpuNic,
+                phase: FirmwareUpdatePhase::Started,
+                machine_id: host_machine_id,
+                detail: dpu_update_string,
+            });
             updates_started.insert(host_machine_id);
         }
 
@@ -163,13 +181,26 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
                         machine_id = %updated_machine.dpu_machine_id,
                         "Failed to remove machine update markers: {}", e
                     );
+                } else if let Ok(mut reported) = self.reported_wrong_versions.lock() {
+                    reported.retain(|(dpu, _)| *dpu != updated_machine.dpu_machine_id);
                 }
-            } else {
-                tracing::warn!(
-                    machine_id = %updated_machine.dpu_machine_id,
-                    firmware_version = %updated_machine.firmware_version,
-                    "Incorrect firmware version after attempted update"
-                );
+            } else if self
+                .reported_wrong_versions
+                .lock()
+                .is_ok_and(|mut reported| {
+                    reported.insert((
+                        updated_machine.dpu_machine_id,
+                        updated_machine.firmware_version.clone(),
+                    ))
+                })
+            {
+                carbide_instrument::emit(FirmwareUpdateFailed {
+                    target: FirmwareUpdateTarget::DpuNic,
+                    cause: FirmwareUpdateFailureCause::WrongVersionAfterUpdate,
+                    machine_id: updated_machine.dpu_machine_id,
+                    unmatched_dpu_machine_id: String::new(),
+                    firmware_version: updated_machine.firmware_version,
+                });
             }
         }
         Ok(())
@@ -243,6 +274,7 @@ impl DpuNicFirmwareUpdate {
         let mut metrics = DpuNicFirmwareUpdateMetrics::new();
         metrics.register_callbacks(&meter);
         Some(DpuNicFirmwareUpdate {
+            reported_wrong_versions: std::sync::Mutex::new(HashSet::new()),
             metrics: Some(metrics),
             config,
             dpf,

crates/api-core/src/machine_update_manager/host_firmware.rs

@@ -32,6 +32,7 @@ use sqlx::PgConnection;
 use tokio::sync::Mutex;
 
 use super::machine_update_module::MachineUpdateModule;
+use super::metrics::{FirmwareUpdatePhase, FirmwareUpdateProgress, FirmwareUpdateTarget};
 use crate::CarbideResult;
 use crate::cfg::file::CarbideConfig;
 
@@ -90,15 +91,22 @@ impl MachineUpdateModule for HostFirmwareUpdate {
                 continue;
             }
 
-            tracing::info!("Moving {} to host reprovision", machine_update);
-
             db::host_machine_update::trigger_host_reprovisioning_request(
                 &mut txn,
                 "Automated",
                 machine_update,
             )
             .await?;
 
+            // Counted after the trigger succeeds; the commit below spans the
+            // whole batch, so a later DB error can re-count machines on the
+            // next pass -- the same repeat-on-retry the old log line had.
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::Host,
+                phase: FirmwareUpdatePhase::Started,
+                machine_id: *machine_update,
+                detail: String::new(),
+            });
             updates_started.insert(*machine_update);
         }
 
@@ -109,18 +117,21 @@ impl MachineUpdateModule for HostFirmwareUpdate {
     async fn clear_completed_updates(&self, txn: &mut PgConnection) -> CarbideResult<()> {
         let completed = db::host_machine_update::find_completed_updates(txn).await?;
 
-        if !completed.is_empty() {
-            tracing::info!("Completed host firmware updates: {completed:?}");
-            for machine in completed {
-                db::machine::remove_health_report(
-                    txn,
-                    &machine,
-                    health_report::HealthReportApplyMode::Merge,
-                    HOST_FW_UPDATE_HEALTH_REPORT_SOURCE,
-                )
-                .await?;
-                db::machine::update_update_complete(&machine, true, txn).await?;
-            }
+        for machine in completed {
+            db::machine::remove_health_report(
+                txn,
+                &machine,
+                health_report::HealthReportApplyMode::Merge,
+                HOST_FW_UPDATE_HEALTH_REPORT_SOURCE,
+            )
+            .await?;
+            db::machine::update_update_complete(&machine, true, txn).await?;
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::Host,
+                phase: FirmwareUpdatePhase::Completed,
+                machine_id: machine,
+                detail: String::new(),
+            });
         }
         Ok(())
     }