feat: make PXE boot outcomes visible behind their HTTP 200s

The PXE boot server answers most failures with an HTTP 200 error template, so its `http_*` metrics read healthy while machines fail to boot -- and four of its failure paths do not even log. This makes every boot-path outcome countable.

- `carbide_pxe_boot_outcomes_total{endpoint, reason}` -- one outcome per request across `whoami`, `boot`, and the cloud-init routes: `ok`, `interface_not_found`, `instructions_empty`, `metadata_not_found`, `upstream_api_error`, and `architecture_not_found` (counted at the extractor that actually rejects a bad `buildarch`, since such a request never reaches a handler).
- The `/metrics` endpoint now serves two registries in one scrape: the existing `http_*` family renders byte-identical first, then the framework's metrics follow -- the forked axum metric layer stays untouched, with its OpenTelemetry migration left to the governance work.

Notable details:

- The events are metric-only (`log = off`): the binary has no tracing subscriber, and every existing stderr line stays as it was.
- Reasons are per-site truthful: the cloud-init generic-error funnel takes its reason from the caller, so the same generic template counts as `interface_not_found` or `metadata_not_found` depending on what was actually missing.
- Requests rejected before a handler runs return real 4xx codes the `http_*` metrics already count; only the bad-architecture rejection is additionally counted here, as a boot outcome operators watch for.

Tests added!

This supports #3177

Signed-off-by: Chet Nichols III <chetn@nvidia.com>

Author: chet

Cargo.lock

@@ -29,11 +29,13 @@ path = "src/main.rs"
 [dependencies]
 # [begin-local-dependencies]
 carbide-host-support = { path = "../host-support", default-features = false }
+carbide-instrument = { path = "../instrument" }
 carbide-tls = { path = "../tls" }
 carbide-utils = { path = "../utils", default-features = false }
 carbide-version = { path = "../version" }
 carbide-rpc = { path = "../rpc" }
 carbide-uuid = { path = "../uuid" }
+metrics-endpoint = { path = "../metrics-endpoint" }
 # [end-local-dependencies]
 
 axum = { workspace = true }
@@ -47,6 +49,7 @@ metrics = { workspace = true }
 metrics-exporter-prometheus = { workspace = true }
 mime = { workspace = true }
 pin-project-lite = { workspace = true }
+prometheus = { workspace = true }
 serde = { features = ["derive"], workspace = true }
 tera = { workspace = true }
 tokio = { workspace = true }
@@ -62,6 +65,7 @@ uuid = { features = ["v4", "serde"], workspace = true }
 carbide-version = { path = "../version" }
 
 [dev-dependencies]
+carbide-instrument = { path = "../instrument", features = ["test-support"] }
 carbide-test-support = { path = "../test-support" }
 tempfile = { workspace = true }
 

crates/pxe/Cargo.toml

@@ -50,4 +50,33 @@ pub(crate) struct AppState {
     // pub request_metrics: RequestMetrics,
     pub runtime_config: RuntimeConfig,
     pub prometheus_handle: PrometheusHandle,
+    /// The registry behind the global OTel meter, where the instrumentation
+    /// framework's events record; `/metrics` renders it alongside the
+    /// `metrics-exporter-prometheus` recorder above.
+    pub otel_registry: prometheus::Registry,
+}
+
+/// An [`AppState`] for handler tests: an empty template engine, a local
+/// (uninstalled) recorder, and a fresh OTel registry.
+#[cfg(test)]
+pub(crate) fn test_app_state() -> AppState {
+    use metrics_exporter_prometheus::PrometheusBuilder;
+
+    AppState {
+        engine: Engine::from(Tera::default()),
+        runtime_config: RuntimeConfig {
+            internal_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079".to_string(),
+            client_facing_api_url: "https://carbide-api.forge".to_string(),
+            pxe_url: "http://carbide-pxe.forge".to_string(),
+            static_pxe_url: "http://carbide-pxe.forge".to_string(),
+            forge_root_ca_path: String::new(),
+            server_cert_path: String::new(),
+            server_key_path: String::new(),
+            bind_address: "0.0.0.0".parse().unwrap(),
+            bind_port: 8080,
+            template_directory: String::new(),
+        },
+        prometheus_handle: PrometheusBuilder::new().build_recorder().handle(),
+        otel_registry: prometheus::Registry::new(),
+    }
 }

crates/pxe/src/common.rs

@@ -46,15 +46,28 @@ where
     type Rejection = PxeRequestError;
 
     async fn from_request_parts(parts: &mut Parts, state: &S) -> Result<Self, Self::Rejection> {
+        // A missing or malformed build architecture is a boot outcome
+        // operators watch for, so both rejection shapes count as one --
+        // even though the client gets a real 4xx rather than a template.
+        // This extractor serves only the boot route.
+        let count_bad_architecture = || {
+            carbide_instrument::emit(crate::metrics::PxeBootOutcome {
+                endpoint: crate::metrics::BootEndpoint::Boot,
+                reason: crate::metrics::OutcomeReason::ArchitectureNotFound,
+            });
+        };
+
         let Ok(maybe) = Query::<MaybeMachineInterface>::from_request_parts(parts, state).await
         else {
             // Query parsing only fails on the required build_architecture
             // field; everything else is optional.
+            count_bad_architecture();
             return Err(PxeRequestError::InvalidBuildArch);
         };
         let maybe = maybe.0;
 
-        let build_architecture = MachineArchitecture::try_from(maybe.build_architecture.as_str())?;
+        let build_architecture = MachineArchitecture::try_from(maybe.build_architecture.as_str())
+            .inspect_err(|_| count_bad_architecture())?;
 
         // Note: This does *NOT* look at X-Forwarded-For, due to security issues with the header. We
         // don't currently have use cases for a proxy in front of carbide-pxe... if that changes

crates/pxe/src/extractors/machine_interface.rs

@@ -69,6 +69,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     println!("Start carbide-pxe version {}", carbide_version::version!());
     let prometheus_handle = metrics::setup_prometheus();
 
+    // The instrumentation framework's events resolve their instruments from
+    // the global OTel meter, so it installs before the router (and any first
+    // emit) exists. The returned setup owns the meter provider and must stay
+    // alive for the process lifetime -- dropping it stops collection.
+    let otel_metrics = metrics_endpoint::new_metrics_setup("carbide-pxe", "carbide", true)
+        .expect("unable to install the OTel meter provider?");
+
     let runtime_config =
         config::RuntimeConfig::from_env().expect("unable to build runtime config?");
 
@@ -80,6 +87,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         engine: Engine::from(tera),
         runtime_config,
         prometheus_handle,
+        otel_registry: otel_metrics.registry.clone(),
     };
 
     let app = Router::new()

crates/pxe/src/main.rs

@@ -16,6 +16,7 @@
  */
 use std::time::Duration;
 
+use carbide_instrument::{Event, LabelValue};
 use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
 use tokio::time::sleep;
 
@@ -62,3 +63,178 @@ pub(crate) fn setup_prometheus() -> PrometheusHandle {
 
     prometheus_handle
 }
+
+/// The boot-path endpoint an outcome describes, as a bounded metric label:
+/// the two iPXE script routes plus the cloud-init route family
+/// (user-data, meta-data, vendor-data).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, LabelValue)]
+pub(crate) enum BootEndpoint {
+    Whoami,
+    Boot,
+    CloudInit,
+}
+
+/// How a boot-path request resolved, as a bounded metric label. Every
+/// non-`Ok` variant is a response the machine receives as an error script
+/// or generic error template over HTTP 200 -- this label is what makes
+/// those outcomes visible, since the status-code metrics cannot see them.
+/// Requests rejected before a handler runs (a malformed `buildarch`, an
+/// upstream failure inside the `Machine` extractor) return real 4xx codes
+/// the `http_*` metrics already count; only `architecture_not_found` is
+/// also emitted from its extractor, because a bad architecture is a boot
+/// outcome operators watch for. `upstream_api_error` is therefore
+/// structurally boot-only. `ok` means the request resolved to a servable
+/// response; a template that later fails to render returns a real 5xx the
+/// `http_*` metrics count, which is outside this metric's HTTP-200 scope.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, LabelValue)]
+pub(crate) enum OutcomeReason {
+    Ok,
+    ArchitectureNotFound,
+    InterfaceNotFound,
+    InstructionsEmpty,
+    MetadataNotFound,
+    UpstreamApiError,
+}
+
+/// A boot-path request resolved to a servable response -- the real script
+/// or an error template. Metric-only: the existing stderr lines on the
+/// failure paths stay as they are, and the rate is the signal.
+#[derive(Event)]
+#[event(
+    name = "carbide_pxe_boot_outcomes_total",
+    component = "carbide-pxe",
+    log = off,
+    metric = counter,
+    describe = "Number of PXE boot-path outcomes served, by endpoint and reason."
+)]
+pub(crate) struct PxeBootOutcome {
+    #[label]
+    pub endpoint: BootEndpoint,
+    #[label]
+    pub reason: OutcomeReason,
+}
+
+#[cfg(test)]
+mod tests {
+    use carbide_instrument::emit;
+    use carbide_instrument::testing::{MetricsCapture, capture_logs};
+    use carbide_test_support::{Check, check_values};
+
+    use super::*;
+
+    /// The label vocabulary is the dashboard contract: each variant renders
+    /// as its snake_case name, byte for byte.
+    #[test]
+    fn label_values_render_as_snake_case() {
+        check_values(
+            [
+                Check {
+                    scenario: "whoami endpoint",
+                    input: BootEndpoint::Whoami.label_value(),
+                    expect: "whoami".to_string(),
+                },
+                Check {
+                    scenario: "boot endpoint",
+                    input: BootEndpoint::Boot.label_value(),
+                    expect: "boot".to_string(),
+                },
+                Check {
+                    scenario: "cloud-init endpoint",
+                    input: BootEndpoint::CloudInit.label_value(),
+                    expect: "cloud_init".to_string(),
+                },
+                Check {
+                    scenario: "ok",
+                    input: OutcomeReason::Ok.label_value(),
+                    expect: "ok".to_string(),
+                },
+                Check {
+                    scenario: "architecture not found",
+                    input: OutcomeReason::ArchitectureNotFound.label_value(),
+                    expect: "architecture_not_found".to_string(),
+                },
+                Check {
+                    scenario: "interface not found",
+                    input: OutcomeReason::InterfaceNotFound.label_value(),
+                    expect: "interface_not_found".to_string(),
+                },
+                Check {
+                    scenario: "instructions empty",
+                    input: OutcomeReason::InstructionsEmpty.label_value(),
+                    expect: "instructions_empty".to_string(),
+                },
+                Check {
+                    scenario: "upstream API error",
+                    input: OutcomeReason::UpstreamApiError.label_value(),
+                    expect: "upstream_api_error".to_string(),
+                },
+                Check {
+                    scenario: "render failure",
+                    input: OutcomeReason::MetadataNotFound.label_value(),
+                    expect: "metadata_not_found".to_string(),
+                },
+            ],
+            |value| value.to_string(),
+        );
+    }
+
+    /// Each emit moves exactly its label pair's series, and none of them
+    /// builds a log line -- the event is declared `log = off` because this
+    /// binary has no tracing subscriber and its stderr lines stay untouched.
+    #[test]
+    fn boot_outcomes_count_per_label_without_logging() {
+        let metrics = MetricsCapture::start();
+        let logs = capture_logs(|| {
+            emit(PxeBootOutcome {
+                endpoint: BootEndpoint::Whoami,
+                reason: OutcomeReason::Ok,
+            });
+            emit(PxeBootOutcome {
+                endpoint: BootEndpoint::Boot,
+                reason: OutcomeReason::UpstreamApiError,
+            });
+            emit(PxeBootOutcome {
+                endpoint: BootEndpoint::Boot,
+                reason: OutcomeReason::UpstreamApiError,
+            });
+            emit(PxeBootOutcome {
+                endpoint: BootEndpoint::CloudInit,
+                reason: OutcomeReason::MetadataNotFound,
+            });
+        });
+
+        assert!(
+            logs.is_empty(),
+            "log = off must not construct any log line, got {logs:?}"
+        );
+        assert_eq!(
+            metrics.counter_delta(
+                "carbide_pxe_boot_outcomes_total",
+                &[("endpoint", "whoami"), ("reason", "ok")],
+            ),
+            1.0,
+        );
+        assert_eq!(
+            metrics.counter_delta(
+                "carbide_pxe_boot_outcomes_total",
+                &[("endpoint", "boot"), ("reason", "upstream_api_error")],
+            ),
+            2.0,
+        );
+        assert_eq!(
+            metrics.counter_delta(
+                "carbide_pxe_boot_outcomes_total",
+                &[("endpoint", "cloud_init"), ("reason", "metadata_not_found")],
+            ),
+            1.0,
+        );
+        assert_eq!(
+            metrics.counter_delta(
+                "carbide_pxe_boot_outcomes_total",
+                &[("endpoint", "boot"), ("reason", "ok")],
+            ),
+            0.0,
+            "an untouched label pair must not move",
+        );
+    }
+}