feat: count firmware update outcomes across every target

Firmware updates run for three targets -- host BIOS/BMC, DPU NICs, and SuperNICs -- and none of their outcomes are countable today: starts, completions, and failures live only in log lines, including the "wrong version after attempted update" case that means an update silently didn't take. This makes all of it countable, with each event replacing its site's log line at the historical level (info for progress, warn for failures) and the same information as structured fields.

Two counters from `carbide-instrument`:

- `carbide_firmware_updates_total{target, phase}` -- started and completed, per target. For hosts both phases fire, so the started-to-completed gap is the in-flight backlog; the DPU-NIC and SuperNIC targets emit started only, with completion visible in their own state flows.
- `carbide_firmware_update_failures_total{target, cause}` -- `no_update_match` (a DPU left ready between snapshot and trigger) and `wrong_version_after_update` (the update ran and the firmware version still doesn't match -- the silent-failure case).

Notable details:

- Started counts only committed triggers: the DPU-NIC emit fires after `txn.commit()`, so an update whose trigger rolled back is a `no_update_match` failure, never a started update, and the started-minus-completed math stays truthful.
- Host completion emits once per machine as its update markers clear, replacing one Vec-debug log line with per-machine lines.
- A SuperNIC re-counts one started per scout pass while a device stays mismatched -- each pass issues a real apply command, and the sustained rate is the stuck-device signal.

Tests added!

This supports #3175

Signed-off-by: Chet Nichols III <chetn@nvidia.com>

Author: chet

crates/api-core/src/handlers/svpc.rs

@@ -34,6 +34,9 @@ use rpc::protos::mlx_device::MlxDeviceInfo;
 use tonic::{Request, Response, Status};
 
 use crate::api::{Api, log_request_data};
+use crate::machine_update_manager::metrics::{
+    FirmwareUpdatePhase, FirmwareUpdateProgress, FirmwareUpdateTarget,
+};
 use crate::{CarbideError, CarbideResult};
 
 // Code to handle SVPC specific information.
@@ -197,12 +200,16 @@ fn build_apply_firmware_command<'a>(
             return None;
         }
 
-        tracing::info!(
-            %machine_id, %pci_name, %part_number, %psid,
-            observed_fw_version = ?device_info.fw_version_current,
-            expected_fw_version = %fw_profile.firmware_spec.version,
-            "firmware version mismatch, applying firmware"
-        );
+        carbide_instrument::emit(FirmwareUpdateProgress {
+            target: FirmwareUpdateTarget::SuperNic,
+            phase: FirmwareUpdatePhase::Started,
+            machine_id,
+            detail: format!(
+                "pci_name={pci_name} part_number={part_number} psid={psid} \
+                 observed_fw_version={:?} expected_fw_version={}",
+                device_info.fw_version_current, fw_profile.firmware_spec.version
+            ),
+        });
         Some(Cow::Borrowed(fw_profile))
     })();
 

crates/api-core/src/machine_update_manager/dpu_nic_firmware.rs

@@ -29,6 +29,10 @@ use sqlx::PgConnection;
 
 use super::dpu_nic_firmware_metrics::DpuNicFirmwareUpdateMetrics;
 use super::machine_update_module::MachineUpdateModule;
+use super::metrics::{
+    FirmwareUpdateFailed, FirmwareUpdateFailureCause, FirmwareUpdatePhase, FirmwareUpdateProgress,
+    FirmwareUpdateTarget,
+};
 use crate::cfg::file::CarbideConfig;
 use crate::machine_update_manager::MachineUpdateManager;
 use crate::{CarbideResult, DatabaseError};
@@ -105,11 +109,6 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
                 output + format!("{} ({}) ", dpu.dpu_machine_id, dpu.firmware_version).as_str()
             });
 
-            tracing::info!(
-                "Starting DPU updates for host {}: {}",
-                host_machine_id,
-                dpu_update_string
-            );
             // If the reprovisioning failed to update the database for a
             // given {dpu,host}_machine_id, log it as a warning and don't
             // add it to updates_started.
@@ -124,11 +123,13 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
             {
                 match reprovisioning_err {
                     DatabaseError::NotFoundError { id, .. } => {
-                        tracing::warn!(
-                            "failed to trigger reprovisioning for managed host : {} - no update match for id: {}",
-                            host_machine_id,
-                            id
-                        );
+                        carbide_instrument::emit(FirmwareUpdateFailed {
+                            target: FirmwareUpdateTarget::DpuNic,
+                            cause: FirmwareUpdateFailureCause::NoUpdateMatch,
+                            machine_id: host_machine_id,
+                            unmatched_dpu_machine_id: id,
+                            firmware_version: String::new(),
+                        });
                         continue;
                     }
                     _ => {
@@ -139,6 +140,15 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
 
             txn.commit().await?;
 
+            // Counted only once the trigger is committed: a DPU that left
+            // ready between snapshot and trigger is a NoUpdateMatch failure,
+            // not a started update.
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::DpuNic,
+                phase: FirmwareUpdatePhase::Started,
+                machine_id: host_machine_id,
+                detail: dpu_update_string,
+            });
             updates_started.insert(host_machine_id);
         }
 
@@ -165,11 +175,13 @@ impl MachineUpdateModule for DpuNicFirmwareUpdate {
                     );
                 }
             } else {
-                tracing::warn!(
-                    machine_id = %updated_machine.dpu_machine_id,
-                    firmware_version = %updated_machine.firmware_version,
-                    "Incorrect firmware version after attempted update"
-                );
+                carbide_instrument::emit(FirmwareUpdateFailed {
+                    target: FirmwareUpdateTarget::DpuNic,
+                    cause: FirmwareUpdateFailureCause::WrongVersionAfterUpdate,
+                    machine_id: updated_machine.dpu_machine_id,
+                    unmatched_dpu_machine_id: String::new(),
+                    firmware_version: updated_machine.firmware_version,
+                });
             }
         }
         Ok(())

crates/api-core/src/machine_update_manager/host_firmware.rs

@@ -32,6 +32,7 @@ use sqlx::PgConnection;
 use tokio::sync::Mutex;
 
 use super::machine_update_module::MachineUpdateModule;
+use super::metrics::{FirmwareUpdatePhase, FirmwareUpdateProgress, FirmwareUpdateTarget};
 use crate::CarbideResult;
 use crate::cfg::file::CarbideConfig;
 
@@ -90,15 +91,22 @@ impl MachineUpdateModule for HostFirmwareUpdate {
                 continue;
             }
 
-            tracing::info!("Moving {} to host reprovision", machine_update);
-
             db::host_machine_update::trigger_host_reprovisioning_request(
                 &mut txn,
                 "Automated",
                 machine_update,
             )
             .await?;
 
+            // Counted after the trigger succeeds; the commit below spans the
+            // whole batch, so a later DB error can re-count machines on the
+            // next pass -- the same repeat-on-retry the old log line had.
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::Host,
+                phase: FirmwareUpdatePhase::Started,
+                machine_id: *machine_update,
+                detail: String::new(),
+            });
             updates_started.insert(*machine_update);
         }
 
@@ -109,18 +117,21 @@ impl MachineUpdateModule for HostFirmwareUpdate {
     async fn clear_completed_updates(&self, txn: &mut PgConnection) -> CarbideResult<()> {
         let completed = db::host_machine_update::find_completed_updates(txn).await?;
 
-        if !completed.is_empty() {
-            tracing::info!("Completed host firmware updates: {completed:?}");
-            for machine in completed {
-                db::machine::remove_health_report(
-                    txn,
-                    &machine,
-                    health_report::HealthReportApplyMode::Merge,
-                    HOST_FW_UPDATE_HEALTH_REPORT_SOURCE,
-                )
-                .await?;
-                db::machine::update_update_complete(&machine, true, txn).await?;
-            }
+        for machine in completed {
+            db::machine::remove_health_report(
+                txn,
+                &machine,
+                health_report::HealthReportApplyMode::Merge,
+                HOST_FW_UPDATE_HEALTH_REPORT_SOURCE,
+            )
+            .await?;
+            db::machine::update_update_complete(&machine, true, txn).await?;
+            carbide_instrument::emit(FirmwareUpdateProgress {
+                target: FirmwareUpdateTarget::Host,
+                phase: FirmwareUpdatePhase::Completed,
+                machine_id: machine,
+                detail: String::new(),
+            });
         }
         Ok(())
     }