feat: count firmware update outcomes across every target

crates/api-core/src/handlers/host_reprovisioning.rs

@@ -79,6 +79,17 @@ pub(crate) async fn trigger_host_reprovisioning(
                 &machine_id,
             )
             .await?;
+            // Manual initiations pair with the same completion emit the
+            // update manager's automatic path gets, keeping the
+            // started-to-completed gap truthful for every initiator.
+            carbide_instrument::emit(
+                crate::machine_update_manager::metrics::FirmwareUpdateProgress {
+                    target: crate::machine_update_manager::metrics::FirmwareUpdateTarget::Host,
+                    phase: crate::machine_update_manager::metrics::FirmwareUpdatePhase::Started,
+                    machine_id,
+                    detail: initiator.to_string(),
+                },
+            );
         }
         Mode::Clear => {
             db::host_machine_update::clear_host_reprovisioning_request(&mut txn, &machine_id)

crates/api-core/src/handlers/svpc.rs

@@ -34,6 +34,9 @@ use rpc::protos::mlx_device::MlxDeviceInfo;
 use tonic::{Request, Response, Status};
 
 use crate::api::{Api, log_request_data};
+use crate::machine_update_manager::metrics::{
+    FirmwareUpdatePhase, FirmwareUpdateProgress, FirmwareUpdateTarget,
+};
 use crate::{CarbideError, CarbideResult};
 
 // Code to handle SVPC specific information.
@@ -197,12 +200,16 @@ fn build_apply_firmware_command<'a>(
             return None;
         }
 
-        tracing::info!(
-            %machine_id, %pci_name, %part_number, %psid,
-            observed_fw_version = ?device_info.fw_version_current,
-            expected_fw_version = %fw_profile.firmware_spec.version,
-            "firmware version mismatch, applying firmware"
-        );
+        carbide_instrument::emit(FirmwareUpdateProgress {
+            target: FirmwareUpdateTarget::SuperNic,
+            phase: FirmwareUpdatePhase::Started,
+            machine_id,
+            detail: format!(
+                "pci_name={pci_name} part_number={part_number} psid={psid} \
+                 observed_fw_version={:?} expected_fw_version={}",
+                device_info.fw_version_current, fw_profile.firmware_spec.version
+            ),
+        });
         Some(Cow::Borrowed(fw_profile))
     })();
 

crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs

@@ -29,6 +29,10 @@ use sqlx::PgConnection;
 
 use super::dpu_nic_firmware_metrics::DpuNicFirmwareUpdateMetrics;
 use super::machine_update_module::MachineUpdateModule;
+use super::metrics::{
+    FirmwareUpdateFailed, FirmwareUpdateFailureCause, FirmwareUpdatePhase, FirmwareUpdateProgress,
+    FirmwareUpdateTarget,
+};
 use crate::cfg::file::CarbideConfig;
 use crate::machine_update_manager::MachineUpdateManager;
 use crate::{CarbideResult, DatabaseError};
@@ -46,6 +50,14 @@ pub struct DpuNicFirmwareUpdate {
     /// DPF handle for discovering outdated DPF-managed DPUs. `None` when DPF
     /// is disabled in config; in that case `find_outdated_dpus_dpf` is not called.
     pub dpf: Option<Arc<dyn DpfOperations>>,
+    /// Wrong-version outcomes already counted, keyed by DPU and the version it
+    /// landed on: the condition persists across manager passes (the update
+    /// marker stays until an operator intervenes or a new update runs), and a
+    /// counter must record the outcome once, not once per poll. An entry
+    /// clears when the DPU's markers clear, so a later attempt that lands
+    /// wrong again is a new outcome. In-memory: a restart re-reports at most
+    /// once per stuck DPU.
+    pub reported_wrong_versions: std::sync::Mutex<HashSet<(MachineId, String)>>,
 }
 
 #[async_trait]
@@ -105,11 +117,6 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
                 output + format!("{} ({}) ", dpu.dpu_machine_id, dpu.firmware_version).as_str()
             });
 
-            tracing::info!(
-                "Starting DPU updates for host {}: {}",
-                host_machine_id,
-                dpu_update_string
-            );
             // If the reprovisioning failed to update the database for a
             // given {dpu,host}_machine_id, log it as a warning and don't
             // add it to updates_started.
@@ -124,11 +131,13 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
             {
                 match reprovisioning_err {
                     DatabaseError::NotFoundError { id, .. } => {
-                        tracing::warn!(
-                            "failed to trigger reprovisioning for managed host : {} - no update match for id: {}",
-                            host_machine_id,
-                            id
-                        );
+                        carbide_instrument::emit(FirmwareUpdateFailed {
+                            target: FirmwareUpdateTarget::DpuNic,
+                            cause: FirmwareUpdateFailureCause::NoUpdateMatch,
+                            machine_id: host_machine_id,
+                            unmatched_dpu_machine_id: id,
+                            firmware_version: String::new(),
+                        });
                         continue;
                     }
                     _ => {
@@ -139,6 +148,15 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
 
             txn.commit().await?;
 
+            // Counted only once the trigger is committed: a DPU that left
+            // ready between snapshot and trigger is a NoUpdateMatch failure,
+            // not a started update.
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::DpuNic,
+                phase: FirmwareUpdatePhase::Started,
+                machine_id: host_machine_id,
+                detail: dpu_update_string,
+            });
             updates_started.insert(host_machine_id);
         }
 
@@ -163,13 +181,26 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
                         machine_id = %updated_machine.dpu_machine_id,
                         "Failed to remove machine update markers: {}", e
                     );
+                } else if let Ok(mut reported) = self.reported_wrong_versions.lock() {
+                    reported.retain(|(dpu, _)| *dpu != updated_machine.dpu_machine_id);
                 }
-            } else {
-                tracing::warn!(
-                    machine_id = %updated_machine.dpu_machine_id,
-                    firmware_version = %updated_machine.firmware_version,
-                    "Incorrect firmware version after attempted update"
-                );
+            } else if self
+                .reported_wrong_versions
+                .lock()
+                .is_ok_and(|mut reported| {
+                    reported.insert((
+                        updated_machine.dpu_machine_id,
+                        updated_machine.firmware_version.clone(),
+                    ))
+                })
+            {
+                carbide_instrument::emit(FirmwareUpdateFailed {
+                    target: FirmwareUpdateTarget::DpuNic,
+                    cause: FirmwareUpdateFailureCause::WrongVersionAfterUpdate,
+                    machine_id: updated_machine.dpu_machine_id,
+                    unmatched_dpu_machine_id: String::new(),
+                    firmware_version: updated_machine.firmware_version,
+                });
             }
         }
         Ok(())
@@ -243,6 +274,7 @@ impl DpuNicFirmwareUpdate {
         let mut metrics = DpuNicFirmwareUpdateMetrics::new();
         metrics.register_callbacks(&meter);
         Some(DpuNicFirmwareUpdate {
+            reported_wrong_versions: std::sync::Mutex::new(HashSet::new()),
             metrics: Some(metrics),
             config,
             dpf,

crates/api-core/src/machine_update_manager/host_firmware.rs

@@ -32,6 +32,7 @@ use sqlx::PgConnection;
 use tokio::sync::Mutex;
 
 use super::machine_update_module::MachineUpdateModule;
+use super::metrics::{FirmwareUpdatePhase, FirmwareUpdateProgress, FirmwareUpdateTarget};
 use crate::CarbideResult;
 use crate::cfg::file::CarbideConfig;
 
@@ -90,15 +91,22 @@ impl MachineUpdateModule for HostFirmwareUpdate {
                 continue;
             }
 
-            tracing::info!("Moving {} to host reprovision", machine_update);
-
             db::host_machine_update::trigger_host_reprovisioning_request(
                 &mut txn,
                 "Automated",
                 machine_update,
             )
             .await?;
 
+            // Counted after the trigger succeeds; the commit below spans the
+            // whole batch, so a later DB error can re-count machines on the
+            // next pass -- the same repeat-on-retry the old log line had.
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::Host,
+                phase: FirmwareUpdatePhase::Started,
+                machine_id: *machine_update,
+                detail: String::new(),
+            });
             updates_started.insert(*machine_update);
         }
 
@@ -109,18 +117,21 @@ impl MachineUpdateModule for HostFirmwareUpdate {
     async fn clear_completed_updates(&self, txn: &mut PgConnection) -> CarbideResult<()> {
         let completed = db::host_machine_update::find_completed_updates(txn).await?;
 
-        if !completed.is_empty() {
-            tracing::info!("Completed host firmware updates: {completed:?}");
-            for machine in completed {
-                db::machine::remove_health_report(
-                    txn,
-                    &machine,
-                    health_report::HealthReportApplyMode::Merge,
-                    HOST_FW_UPDATE_HEALTH_REPORT_SOURCE,
-                )
-                .await?;
-                db::machine::update_update_complete(&machine, true, txn).await?;
-            }
+        for machine in completed {
+            db::machine::remove_health_report(
+                txn,
+                &machine,
+                health_report::HealthReportApplyMode::Merge,
+                HOST_FW_UPDATE_HEALTH_REPORT_SOURCE,
+            )
+            .await?;
+            db::machine::update_update_complete(&machine, true, txn).await?;
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::Host,
+                phase: FirmwareUpdatePhase::Completed,
+                machine_id: machine,
+                detail: String::new(),
+            });
         }
         Ok(())
     }