fix: disable endpoint refresh functionality + remove worklock dependency (#2890)

### Problem

PR #1635 (adhoc endpoint exploration) added a per-endpoint `WorkLock`
that the periodic site-explorer loop acquired before probing each BMC.
This doesn't scale on large sites.

### Change

Back out the work-lock usage introduced for the adhoc exploration
feature; it shouldn't touch the work-lock manager at all:

- **Periodic loop:** removed the per-endpoint work-lock acquire/skip.
- **Adhoc refresh RPC (`RefreshEndpointReport`):** short-circuited to
return gRPC `Unavailable` ("Endpoint refresh is temporarily
unavailable") for now.
- Removed the now-dead `endpoint_exploration_work_key` helper and the
lock-coordination tests; replaced the refresh tests with one asserting
the disabled behavior.

### Notes

- Any orphaned `endpoint_exploration::*` lock rows from the prior
version are never read by the new code and age out via keepalive
timeout.
- Proto is unchanged; the refresh CLI/UI surface the `Unavailable` error
cleanly (no panic, no DB write). The UI Refresh button still appears and
will error until the feature is re-enabled.
## Related issues
<!-- Refer to existing GitHub issues here -->

## Type of Change
<!-- Check one that best describes this PR -->
- [ ] **Add** - New feature or capability
- [ ] **Change** - Changes in existing functionality
- [x] **Fix** - Bug fixes
- [ ] **Remove** - Removed features or deprecated functionality
- [ ] **Internal** - Internal changes (refactoring, tests, docs, etc.)

## Breaking Changes
<!-- If checked, describe the breaking changes and migration steps -->
<!-- Breaking changes are not generally permitted, please discuss on a
GitHub discussion or with the development team if you believe you need
to break a backward compatibility guarantee -->
- [ ] **This PR contains breaking changes**

## Testing
<!-- How was this tested? Check all that apply -->
- [x] Unit tests added/updated
- [ ] Integration tests added/updated
- [ ] Manual testing performed
- [ ] No testing required (docs, internal refactor, etc.)

## Additional Notes
<!-- Any additional context, deployment notes, or reviewer guidance -->

Signed-off-by: Krish Dandiwala <kdandiwala@nvidia.com>

Author: krish-nvidia

crates/api-core/src/handlers/site_explorer.rs

@@ -19,9 +19,8 @@ use std::net::{IpAddr, SocketAddr};
 use std::str::FromStr;
 
 use ::rpc::forge::{self as rpc, IsBmcInManagedHostResponse};
-use carbide_site_explorer::{endpoint_exploration_work_key, enrich_endpoint_exploration_report};
+use carbide_site_explorer::enrich_endpoint_exploration_report;
 use config_version::ConfigVersion;
-use db::work_lock_manager::AcquireLockError;
 use model::expected_entity::ExpectedEntity;
 use tokio::net::lookup_host;
 use tonic::{Request, Response, Status};
@@ -305,11 +304,20 @@ pub(crate) async fn re_explore_endpoint(
     Ok(Response::new(()))
 }
 
+// Short-circuited: adhoc endpoint refresh is temporarily disabled. The probe
+// path below is retained but unreachable until the feature is re-enabled.
+#[allow(unreachable_code, unused_variables)]
 pub(crate) async fn refresh_endpoint_report(
     api: &Api,
     request: Request<rpc::RefreshEndpointReportRequest>,
 ) -> Result<Response<::rpc::site_explorer::ExploredEndpoint>, tonic::Status> {
     log_request_data(&request);
+
+    return Err(CarbideError::UnavailableError(
+        "Endpoint refresh is temporarily unavailable".to_string(),
+    )
+    .into());
+
     let req = request.into_inner();
 
     let bmc_ip = IpAddr::from_str(&req.ip_address).map_err(CarbideError::from)?;
@@ -352,39 +360,14 @@ pub(crate) async fn refresh_endpoint_report(
             .map(ExpectedEntity::PowerShelf)
     };
 
-    // Acquire the per-endpoint work lock before probing. If the periodic site-explorer
-    // loop (or another concurrent refresh) is already probing this endpoint, return an
-    // error immediately rather than running a redundant Redfish call.
-    let work_lock = match api
-        .work_lock_manager_handle
-        .try_acquire_lock(endpoint_exploration_work_key(bmc_ip))
-        .await
-    {
-        Ok(lock) => lock,
-        Err(AcquireLockError::WorkAlreadyLocked(_)) => {
-            return Err(CarbideError::AlreadyInProgress(format!(
-                "Endpoint refresh already in progress for {bmc_ip}"
-            ))
-            .into());
-        }
-        Err(e) => {
-            return Err(CarbideError::internal(format!(
-                "Failed to acquire endpoint work lock for {bmc_ip}: {e}"
-            ))
-            .into());
-        }
-    };
-
-    // Run the probe + persist on a detached tokio task that owns the work lock.
-    // Awaiting the JoinHandle preserves the synchronous UX. Even if the caller navigates
-    // away mid-fetch, the probe will still run to completion.
+    // Run the probe + persist on a detached tokio task. Awaiting the JoinHandle
+    // preserves the synchronous UX. Even if the caller navigates away mid-fetch,
+    // the probe will still run to completion.
     let endpoint_explorer = api.endpoint_explorer.clone();
     let database_connection = api.database_connection.clone();
     let runtime_config = api.runtime_config.clone();
 
     let join_handle = tokio::spawn(async move {
-        let _work_lock = work_lock;
-
         let start = std::time::Instant::now();
         let result = endpoint_explorer
             .explore_endpoint(

crates/api-core/src/tests/site_explorer.rs

@@ -20,16 +20,13 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use carbide_site_explorer::config::SiteExplorerConfig;
-use carbide_site_explorer::endpoint_exploration_work_key;
 use common::api_fixtures::TestEnv;
 use db::{self, ObjectColumnFilter};
 use ipnetwork::IpNetwork;
 use mac_address::MacAddress;
 use model::hardware_info::HardwareInfo;
 use model::machine::ManagedHostStateSnapshot;
-use model::site_explorer::{
-    Chassis, EndpointExplorationError, EndpointExplorationReport, ExploredEndpoint,
-};
+use model::site_explorer::{Chassis, EndpointExplorationReport};
 use model::test_support::{DpuConfig, ManagedHostConfig};
 use rpc::forge::forge_server::Forge;
 use rpc::{DiscoveryData, DiscoveryInfo, MachineDiscoveryInfo};
@@ -728,87 +725,13 @@ async fn host_bmc_ip(
     Ok(bmc_ip)
 }
 
-async fn explored_endpoint(
-    env: &TestEnv,
-    bmc_ip: IpAddr,
-) -> Result<ExploredEndpoint, Box<dyn std::error::Error>> {
-    let mut txn = env.pool.begin().await?;
-    let endpoint = db::explored_endpoints::find_by_ips(txn.as_mut(), vec![bmc_ip])
-        .await?
-        .into_iter()
-        .next()
-        .unwrap();
-    txn.commit().await?;
-    Ok(endpoint)
-}
-
-fn endpoint_explore_call_count(env: &TestEnv, bmc_ip: IpAddr) -> usize {
-    env.endpoint_explorer
-        .explore_endpoint_calls
-        .lock()
-        .unwrap()
-        .iter()
-        .filter(|ip| **ip == bmc_ip)
-        .count()
-}
-
-#[sqlx_test]
-async fn test_refresh_endpoint_report_bumps_report_version(
-    pool: PgPool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let env = common::api_fixtures::create_test_env(pool.clone()).await;
-    let mh = common::api_fixtures::create_managed_host(&env).await;
-    let bmc_ip = host_bmc_ip(&env, &mh).await?;
-    let initial_version = explored_endpoint(&env, bmc_ip).await?.report_version;
-
-    env.api
-        .refresh_endpoint_report(Request::new(rpc::forge::RefreshEndpointReportRequest {
-            ip_address: bmc_ip.to_string(),
-        }))
-        .await?;
-
-    let refreshed = explored_endpoint(&env, bmc_ip).await?;
-    assert!(
-        refreshed.report_version.version_nr() > initial_version.version_nr(),
-        "refresh should bump report version from {} to a newer version, got {}",
-        initial_version.version_nr(),
-        refreshed.report_version.version_nr()
-    );
-
-    Ok(())
-}
-
-#[sqlx_test]
-async fn test_refresh_endpoint_report_rejects_nonexistent_endpoint(
-    pool: PgPool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let env = common::api_fixtures::create_test_env(pool.clone()).await;
-
-    let err = env
-        .api
-        .refresh_endpoint_report(Request::new(rpc::forge::RefreshEndpointReportRequest {
-            ip_address: "99.99.99.99".to_string(),
-        }))
-        .await
-        .unwrap_err();
-
-    assert_eq!(err.code(), tonic::Code::NotFound);
-
-    Ok(())
-}
-
 #[sqlx_test]
-async fn test_refresh_endpoint_report_rejects_duplicate_refresh(
+async fn test_refresh_endpoint_report_is_unavailable(
     pool: PgPool,
 ) -> Result<(), Box<dyn std::error::Error>> {
     let env = common::api_fixtures::create_test_env(pool.clone()).await;
     let mh = common::api_fixtures::create_managed_host(&env).await;
     let bmc_ip = host_bmc_ip(&env, &mh).await?;
-    let _endpoint_lock = env
-        .api
-        .work_lock_manager_handle
-        .try_acquire_lock(endpoint_exploration_work_key(bmc_ip))
-        .await?;
 
     let err = env
         .api
@@ -818,134 +741,7 @@ async fn test_refresh_endpoint_report_rejects_duplicate_refresh(
         .await
         .unwrap_err();
 
-    assert_eq!(err.code(), tonic::Code::AlreadyExists);
-
-    Ok(())
-}
-
-#[sqlx_test]
-async fn test_refresh_endpoint_report_lock_blocks_periodic_probe(
-    pool: PgPool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let env = common::api_fixtures::create_test_env(pool.clone()).await;
-    let mh = common::api_fixtures::create_managed_host(&env).await;
-    let bmc_ip = host_bmc_ip(&env, &mh).await?;
-
-    env.api
-        .re_explore_endpoint(Request::new(rpc::forge::ReExploreEndpointRequest {
-            ip_address: bmc_ip.to_string(),
-            if_version_match: None,
-        }))
-        .await?;
-
-    let calls_before = endpoint_explore_call_count(&env, bmc_ip);
-    let _endpoint_lock = env
-        .api
-        .work_lock_manager_handle
-        .try_acquire_lock(endpoint_exploration_work_key(bmc_ip))
-        .await?;
-
-    env.run_site_explorer_iteration().await;
-
-    assert_eq!(
-        endpoint_explore_call_count(&env, bmc_ip),
-        calls_before,
-        "periodic site explorer probe should be skipped while refresh lock is held"
-    );
-
-    Ok(())
-}
-
-#[sqlx_test]
-async fn test_refresh_endpoint_report_failure_persists_error_and_bumps_version(
-    pool: PgPool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let env = common::api_fixtures::create_test_env(pool.clone()).await;
-    let mh = common::api_fixtures::create_managed_host(&env).await;
-    let bmc_ip = host_bmc_ip(&env, &mh).await?;
-    let initial_version = explored_endpoint(&env, bmc_ip).await?.report_version;
-    env.endpoint_explorer.insert_endpoint_result(
-        bmc_ip,
-        Err(EndpointExplorationError::Unreachable {
-            details: Some("refresh failure".to_string()),
-        }),
-    );
-    env.api
-        .refresh_endpoint_report(Request::new(rpc::forge::RefreshEndpointReportRequest {
-            ip_address: bmc_ip.to_string(),
-        }))
-        .await?;
-
-    let refreshed = explored_endpoint(&env, bmc_ip).await?;
-    assert!(
-        refreshed.report_version.version_nr() > initial_version.version_nr(),
-        "failed refresh should still bump report version"
-    );
-    assert!(
-        refreshed.report.last_exploration_error.is_some(),
-        "failed refresh should persist the exploration error"
-    );
-
-    Ok(())
-}
-
-#[sqlx_test]
-async fn test_refresh_endpoint_report_clears_pending_requested_exploration(
-    pool: PgPool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let env = common::api_fixtures::create_test_env(pool.clone()).await;
-    let mh = common::api_fixtures::create_managed_host(&env).await;
-    let bmc_ip = host_bmc_ip(&env, &mh).await?;
-
-    env.api
-        .re_explore_endpoint(Request::new(rpc::forge::ReExploreEndpointRequest {
-            ip_address: bmc_ip.to_string(),
-            if_version_match: None,
-        }))
-        .await?;
-    assert!(explored_endpoint(&env, bmc_ip).await?.exploration_requested);
-
-    env.api
-        .refresh_endpoint_report(Request::new(rpc::forge::RefreshEndpointReportRequest {
-            ip_address: bmc_ip.to_string(),
-        }))
-        .await?;
-
-    assert!(
-        !explored_endpoint(&env, bmc_ip).await?.exploration_requested,
-        "refresh should clear the pending requested exploration so the endpoint is not immediately probed again as priority work"
-    );
-
-    Ok(())
-}
-
-#[sqlx_test]
-async fn test_refresh_endpoint_report_lock_is_per_endpoint(
-    pool: PgPool,
-) -> Result<(), Box<dyn std::error::Error>> {
-    let env = common::api_fixtures::create_test_env(pool.clone()).await;
-    let mh_a = common::api_fixtures::create_managed_host(&env).await;
-    let mh_b = common::api_fixtures::create_managed_host(&env).await;
-    let bmc_ip_a = host_bmc_ip(&env, &mh_a).await?;
-    let bmc_ip_b = host_bmc_ip(&env, &mh_b).await?;
-    let initial_version_b = explored_endpoint(&env, bmc_ip_b).await?.report_version;
-    let _endpoint_lock = env
-        .api
-        .work_lock_manager_handle
-        .try_acquire_lock(endpoint_exploration_work_key(bmc_ip_a))
-        .await?;
-
-    env.api
-        .refresh_endpoint_report(Request::new(rpc::forge::RefreshEndpointReportRequest {
-            ip_address: bmc_ip_b.to_string(),
-        }))
-        .await?;
-
-    let refreshed_b = explored_endpoint(&env, bmc_ip_b).await?;
-    assert!(
-        refreshed_b.report_version.version_nr() > initial_version_b.version_nr(),
-        "lock for endpoint {bmc_ip_a} should not block refresh for endpoint {bmc_ip_b}"
-    );
+    assert_eq!(err.code(), tonic::Code::Unavailable);
 
     Ok(())
 }

crates/site-explorer/src/lib.rs

@@ -76,7 +76,7 @@ pub use machine_creator::MachineCreator;
 pub mod explored_endpoint_index;
 mod managed_host;
 use db::ObjectColumnFilter;
-use db::work_lock_manager::{AcquireLockError, WorkLockManagerHandle};
+use db::work_lock_manager::WorkLockManagerHandle;
 pub use managed_host::is_endpoint_in_managed_host;
 use model::DpuModel;
 use model::expected_machine::DpuMode;
@@ -253,14 +253,6 @@ impl<'a> Endpoint<'a> {
 
 pub type SiteIdentifiedHosts = Vec<(ExploredManagedHost, EndpointExplorationReport)>;
 
-/// Work-lock key for a single endpoint exploration.
-///
-/// Both the site-explorer loop (`update_explored_endpoints`) and the adhoc
-/// `RefreshEndpointReport` handler acquire this key before probing Redfish.
-pub fn endpoint_exploration_work_key(bmc_ip: IpAddr) -> String {
-    format!("SiteExplorer::endpoint_exploration::{bmc_ip}")
-}
-
 /// The SiteExplorer periodically runs [modules](machine_update_module::MachineUpdateModule) to initiate upgrades of machine components.
 /// On each iteration the SiteExplorer will:
 /// 1. collect the number of outstanding updates from all modules.
@@ -1994,7 +1986,6 @@ impl SiteExplorer {
             let bmc_target_addr = SocketAddr::new(endpoint.address, bmc_target_port);
             let fw_config_snapshot = fw_config_snapshot.clone();
             let database_connection = self.database_connection.clone();
-            let work_lock_manager_handle = self.work_lock_manager_handle.clone();
 
             task_set.push(
                 async move {
@@ -2010,26 +2001,6 @@ impl SiteExplorer {
                         .await
                         .expect("Semaphore can't be closed");
 
-                    // If the endpoint is locked, we skip exploration.
-                    let work_key = endpoint_exploration_work_key(endpoint.address);
-                    let _work_lock = match work_lock_manager_handle.try_acquire_lock(work_key).await
-                    {
-                        Ok(work_lock) => work_lock,
-                        Err(AcquireLockError::WorkAlreadyLocked(_)) => {
-                            tracing::info!(
-                                address = %endpoint.address,
-                                "Skipping periodic endpoint exploration; adhoc refresh already in progress"
-                            );
-                            return Ok(None);
-                        }
-                        Err(e) => {
-                            return Err(SiteExplorerError::internal(format!(
-                                "Failed to acquire per-endpoint work lock for {}: {e}",
-                                endpoint.address
-                            )));
-                        }
-                    };
-
                     let mut result = endpoint_explorer
                         .explore_endpoint(
                             bmc_target_addr,