feat: Add Site Explorer run status to admin UI (#2591)

## Description

Surfaces the latest Site Explorer iteration status in the admin Site
Explorer UI so operators do not have to inspect `carbide-api` pod logs
to understand why a run failed.

This PR adds a persisted singleton last-run record for Site Explorer and
threads it through the existing site exploration report path. The admin
explored-endpoint pages now show:

- Last run status: `Success`, `Failed`, or `No Run`
- Start and finish timestamps
- Endpoint exploration counts: attempted, successful, and errored
- The whole-run failure string when Site Explorer fails before
per-endpoint errors can be written, such as missing global credentials

## Type of Change

- [x] **Add** - New feature or capability
- [ ] **Change** - Changes in existing functionality
- [ ] **Fix** - Bug fixes
- [ ] **Remove** - Removed features or deprecated functionality
- [ ] **Internal** - Internal changes (refactoring, tests, docs, etc.)

## Related Issues

Fixes #2046

## Breaking Changes

- [ ] This PR contains breaking changes

No breaking API behavior is expected. The RPC report gains an optional
`last_run` field, and the database migration adds a new singleton table
used only for the latest Site Explorer run metadata.

## Implementation Notes

- Adds `SiteExplorerLastRun` to the API model and Site Explorer report
RPC.
- Adds `site_explorer_run_status` storage with upsert/fetch helpers.
- Records run metadata at the end of each `run_single_iteration`,
including failures that occur before endpoint exploration starts.
- Keeps failure-to-record status non-fatal so a reporting write problem
does not mask the original Site Explorer result.
- Adds a reusable admin template for the last-run status block and
includes it on all explored-endpoint list views.

## Testing

- [ ] Unit tests added/updated
- [x] Integration tests added/updated
- [ ] Manual testing performed
- [ ] No testing required (docs, internal refactor, etc.)

Validated with:

```bash
cargo fmt --all
cargo fmt --all -- --check
cargo make check-format-nightly
git diff --check
cargo test -p carbide-site-explorer test_site_explorer_records_last_run -- --nocapture
cargo test -p carbide-api-web test_site_explorer_run_status_banner -- --nocapture
```

The database-backed tests were run against a disposable Postgres 14.5
container with `DATABASE_URL` set for the SQLx test harness.

---------

Signed-off-by: Hasan Khan <hasank@nvidia.com>

Author: osu

crates/admin-cli/src/rpc.rs

@@ -664,6 +664,7 @@ impl ApiClient {
         &self,
         page_size: usize,
     ) -> CarbideCliResult<::rpc::site_explorer::SiteExplorationReport> {
+        let last_run = self.get_site_explorer_last_run().await?;
         // grab endpoints
         let endpoint_ids = match self.0.find_explored_endpoint_ids().await {
             Ok(endpoint_ids) => endpoint_ids,
@@ -698,9 +699,20 @@ impl ApiClient {
         Ok(::rpc::site_explorer::SiteExplorationReport {
             endpoints: all_endpoints.endpoints,
             managed_hosts: all_hosts,
+            last_run,
         })
     }
 
+    pub async fn get_site_explorer_last_run(
+        &self,
+    ) -> CarbideCliResult<Option<::rpc::site_explorer::SiteExplorerLastRun>> {
+        match self.0.get_site_explorer_last_run().await {
+            Ok(response) => Ok(response.last_run),
+            Err(status) if maybe_unimplemented(&status) => Ok(None),
+            Err(status) => Err(status.into()),
+        }
+    }
+
     pub async fn get_explored_endpoints_by_ids(
         &self,
         endpoint_ids: &[String],

crates/admin-cli/src/site_explorer/get_report/cmd.rs

@@ -184,6 +184,7 @@ async fn get_exploration_report_for_bmc_address(
     Ok(::rpc::site_explorer::SiteExplorationReport {
         endpoints: endpoints.endpoints,
         managed_hosts: managed_host,
+        last_run: None,
     })
 }
 

crates/api-core/src/api.rs

@@ -1064,6 +1064,13 @@ impl Forge for Api {
         crate::handlers::site_explorer::get_site_exploration_report(self, request).await
     }
 
+    async fn get_site_explorer_last_run(
+        &self,
+        request: Request<()>,
+    ) -> Result<Response<::rpc::site_explorer::SiteExplorerLastRunResponse>, Status> {
+        crate::handlers::site_explorer::get_site_explorer_last_run(self, request).await
+    }
+
     async fn find_explored_endpoint_ids(
         &self,
         request: Request<::rpc::site_explorer::ExploredEndpointSearchFilter>,

crates/api-core/src/auth/internal_rbac_rules.rs

@@ -270,6 +270,7 @@ impl InternalRBACRules {
             "GetSiteExplorationReport",
             vec![ForgeAdminCLI, Machineatron],
         );
+        x.perm("GetSiteExplorerLastRun", vec![ForgeAdminCLI, Machineatron]);
         x.perm("ClearSiteExplorationError", vec![ForgeAdminCLI]);
         x.perm("IsBmcInManagedHost", vec![ForgeAdminCLI]);
         x.perm("Explore", vec![ForgeAdminCLI, Flow]);

crates/api-core/src/handlers/site_explorer.rs

@@ -150,6 +150,21 @@ pub(crate) async fn get_site_exploration_report(
     Ok(tonic::Response::new(report.into()))
 }
 
+pub(crate) async fn get_site_explorer_last_run(
+    api: &Api,
+    request: tonic::Request<()>,
+) -> Result<Response<::rpc::site_explorer::SiteExplorerLastRunResponse>, Status> {
+    log_request_data(&request);
+
+    let last_run = db::site_explorer_run_status::fetch(&mut api.db_reader()).await?;
+
+    Ok(tonic::Response::new(
+        ::rpc::site_explorer::SiteExplorerLastRunResponse {
+            last_run: last_run.map(Into::into),
+        },
+    ))
+}
+
 pub(crate) async fn find_explored_mlx_device_host_ids(
     api: &Api,
     request: Request<::rpc::site_explorer::ExploredMlxDeviceHostSearchFilter>,

crates/api-core/src/tests/common/api_fixtures/mod.rs

@@ -1647,14 +1647,7 @@ pub async fn create_test_env_with_overrides(
         .build_for_manual_iterations(cancel_token.clone())
         .expect("Unable to build RackStateController");
 
-    let fake_endpoint_explorer = MockEndpointExplorer {
-        reports: Arc::new(std::sync::Mutex::new(Default::default())),
-        power_states: Arc::new(std::sync::Mutex::new(Default::default())),
-        redfish_power_control_calls: Arc::new(std::sync::Mutex::new(Default::default())),
-        power_control_failures: Arc::new(std::sync::Mutex::new(Default::default())),
-        set_nic_mode_calls: Arc::new(std::sync::Mutex::new(Default::default())),
-        explore_endpoint_calls: Arc::new(std::sync::Mutex::new(Default::default())),
-    };
+    let fake_endpoint_explorer = MockEndpointExplorer::default();
 
     // The API server is launched with a disabled site-explorer config so that it doesn't launch one
     // on its own. TestEnv's site_explorer is a separate instance talking to the same database that

crates/api-db/migrations/20260614120000_site_explorer_run_status.sql

@@ -0,0 +1,29 @@
+-- Store the operator-facing result of the most recent Site Explorer iteration.
+-- Endpoint rows already hold per-endpoint exploration errors; this singleton
+-- captures whole-run failures such as missing global credentials or database
+-- setup issues that otherwise only appear in nico-api logs.
+CREATE TABLE site_explorer_run_status (
+    id                              smallint    PRIMARY KEY DEFAULT 1 CHECK (id = 1),
+    started_at                      timestamptz NOT NULL,
+    finished_at                     timestamptz NOT NULL,
+    success                         boolean     NOT NULL,
+    error                           text,
+    failure_category                text,
+    endpoint_explorations           bigint      NOT NULL,
+    endpoint_explorations_success   bigint      NOT NULL,
+    endpoint_explorations_failed    bigint      NOT NULL,
+    last_successful_finished_at     timestamptz,
+    last_failed_finished_at         timestamptz,
+    CONSTRAINT site_explorer_run_status_finished_after_started
+        CHECK (finished_at >= started_at),
+    CONSTRAINT site_explorer_run_status_endpoint_explorations_non_negative
+        CHECK (endpoint_explorations >= 0),
+    CONSTRAINT site_explorer_run_status_endpoint_explorations_success_non_negative
+        CHECK (endpoint_explorations_success >= 0),
+    CONSTRAINT site_explorer_run_status_endpoint_explorations_failed_non_negative
+        CHECK (endpoint_explorations_failed >= 0),
+    CONSTRAINT site_explorer_run_status_endpoint_explorations_within_total
+        CHECK (endpoint_explorations_success + endpoint_explorations_failed <= endpoint_explorations),
+    CONSTRAINT site_explorer_run_status_success_has_no_failure_category
+        CHECK (NOT success OR failure_category IS NULL)
+);

crates/api-db/src/lib.rs

@@ -85,6 +85,7 @@ pub mod retained_boot_interface;
 pub mod route_servers;
 pub mod secrets;
 pub mod site_exploration_report;
+pub mod site_explorer_run_status;
 pub mod sku;
 pub mod spx_partition;
 pub mod state_history;

crates/api-db/src/site_exploration_report.rs

@@ -26,8 +26,10 @@ where
     for<'db> &'db mut DB: DbReader<'db>,
 {
     let endpoints = crate::explored_endpoints::find_all(&mut *db).await?;
-    let managed_hosts = crate::explored_managed_host::find_all(db).await?;
+    let managed_hosts = crate::explored_managed_host::find_all(&mut *db).await?;
+    let last_run = crate::site_explorer_run_status::fetch(&mut *db).await?;
     Ok(SiteExplorationReport {
+        last_run,
         endpoints,
         managed_hosts,
     })

crates/api-db/src/site_explorer_run_status.rs

@@ -0,0 +1,133 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use chrono::{DateTime, Utc};
+use model::site_explorer::SiteExplorerLastRun;
+use sqlx::{FromRow, PgConnection};
+
+use crate::db_read::DbReader;
+use crate::{DatabaseError, DatabaseResult};
+
+const LAST_RUN_ID: i16 = 1;
+
+#[derive(Debug, FromRow)]
+struct DbSiteExplorerLastRun {
+    started_at: DateTime<Utc>,
+    finished_at: DateTime<Utc>,
+    success: bool,
+    error: Option<String>,
+    failure_category: Option<String>,
+    endpoint_explorations: i64,
+    endpoint_explorations_success: i64,
+    endpoint_explorations_failed: i64,
+    last_successful_finished_at: Option<DateTime<Utc>>,
+    last_failed_finished_at: Option<DateTime<Utc>>,
+}
+
+impl From<DbSiteExplorerLastRun> for SiteExplorerLastRun {
+    fn from(run: DbSiteExplorerLastRun) -> Self {
+        Self {
+            started_at: run.started_at,
+            finished_at: run.finished_at,
+            success: run.success,
+            error: run.error,
+            failure_category: run.failure_category,
+            endpoint_explorations: run.endpoint_explorations,
+            endpoint_explorations_success: run.endpoint_explorations_success,
+            endpoint_explorations_failed: run.endpoint_explorations_failed,
+            last_successful_finished_at: run.last_successful_finished_at,
+            last_failed_finished_at: run.last_failed_finished_at,
+        }
+    }
+}
+
+/// Fetches metadata for the latest site explorer run.
+pub async fn fetch(db: impl DbReader<'_>) -> DatabaseResult<Option<SiteExplorerLastRun>> {
+    let query = "SELECT started_at, finished_at, success, error, failure_category, endpoint_explorations, endpoint_explorations_success, endpoint_explorations_failed, last_successful_finished_at, last_failed_finished_at
+    FROM site_explorer_run_status
+    WHERE id = $1";
+
+    sqlx::query_as::<_, DbSiteExplorerLastRun>(query)
+        .bind(LAST_RUN_ID)
+        .fetch_optional(db)
+        .await
+        .map(|run| run.map(Into::into))
+        .map_err(|e| DatabaseError::query(query, e))
+}
+
+/// Replaces metadata for the latest site explorer run.
+pub async fn upsert(txn: &mut PgConnection, last_run: &SiteExplorerLastRun) -> DatabaseResult<()> {
+    let query = "INSERT INTO site_explorer_run_status (
+    id,
+    started_at,
+    finished_at,
+    success,
+    error,
+    failure_category,
+    endpoint_explorations,
+    endpoint_explorations_success,
+    endpoint_explorations_failed,
+    last_successful_finished_at,
+    last_failed_finished_at
+)
+VALUES (
+    $1, $2, $3, $4, $5, $6, $7, $8, $9,
+    CASE WHEN $4 THEN $3 ELSE $10 END,
+    CASE WHEN NOT $4 THEN $3 ELSE $11 END
+)
+ON CONFLICT (id) DO UPDATE SET
+    started_at = EXCLUDED.started_at,
+    finished_at = EXCLUDED.finished_at,
+    success = EXCLUDED.success,
+    error = EXCLUDED.error,
+    failure_category = EXCLUDED.failure_category,
+    endpoint_explorations = EXCLUDED.endpoint_explorations,
+    endpoint_explorations_success = EXCLUDED.endpoint_explorations_success,
+    endpoint_explorations_failed = EXCLUDED.endpoint_explorations_failed,
+    last_successful_finished_at = CASE
+        WHEN EXCLUDED.success THEN EXCLUDED.finished_at
+        ELSE COALESCE(
+            EXCLUDED.last_successful_finished_at,
+            site_explorer_run_status.last_successful_finished_at
+        )
+    END,
+    last_failed_finished_at = CASE
+        WHEN NOT EXCLUDED.success THEN EXCLUDED.finished_at
+        ELSE COALESCE(
+            EXCLUDED.last_failed_finished_at,
+            site_explorer_run_status.last_failed_finished_at
+        )
+    END";
+
+    sqlx::query(query)
+        .bind(LAST_RUN_ID)
+        .bind(last_run.started_at)
+        .bind(last_run.finished_at)
+        .bind(last_run.success)
+        .bind(&last_run.error)
+        .bind(&last_run.failure_category)
+        .bind(last_run.endpoint_explorations)
+        .bind(last_run.endpoint_explorations_success)
+        .bind(last_run.endpoint_explorations_failed)
+        .bind(last_run.last_successful_finished_at)
+        .bind(last_run.last_failed_finished_at)
+        .execute(txn)
+        .await
+        .map_err(|e| DatabaseError::query(query, e))?;
+
+    Ok(())
+}