feat: Set NTP servers from site config (#2610)

## Description
- Add site-level `ntp_servers` config.
- Call redfish to `set_ntp_servers` on the BMC during preingestion, see
[libredfish PR87](NVIDIA/libredfish#87).
- Plumb site-level `ntp_servers` config into the `DhcpRecord` response
of `DiscoverDhcp` RPC call.
- Update Kea DHCP config to prefer API-provided `DhcpRecord.ntp_servers`
for Option 42, while keeping the old workflow (read ntp_servers from kea
dhcp config) for fallback.
- Add site level `ntp_servers` into `ManagedHostNetworkConfigResponse`
and have DPU agent use it for DHCP server.

## Type of Change
<!-- Check one that best describes this PR -->
- [x] **Add** - New feature or capability
- [ ] **Change** - Changes in existing functionality  
- [ ] **Fix** - Bug fixes
- [ ] **Remove** - Removed features or deprecated functionality
- [ ] **Internal** - Internal changes (refactoring, tests, docs, etc.)

## Related Issues (Optional)

#548 (comment)

## Breaking Changes
- [ ] This PR contains breaking changes

<!-- If checked above, describe the breaking changes and migration steps
-->

## Testing
<!-- How was this tested? Check all that apply -->
- [x] Unit tests added/updated
- [ ] Integration tests added/updated  
- [x] Manual testing performed
- [ ] No testing required (docs, internal refactor, etc.)

## Additional Notes
<!-- Any additional context, deployment notes, or reviewer guidance -->

---------

Signed-off-by: Felicity Xu <hanyux@nvidia.com>

Author: hanyux-nv

Cargo.lock

@@ -26,7 +26,7 @@ authors = ["NVIDIA Carbide Engineering <carbide-dev@exchange.nvidia.com>"]
 
 [workspace.dependencies]
 clap = { version = "4", features = ["derive", "env"] }
-libredfish = { git = "https://github.com/NVIDIA/libredfish.git", tag = "v0.44.10" }
+libredfish = { git = "https://github.com/NVIDIA/libredfish.git", tag = "v0.44.11" }
 librms = { git = "https://github.com/NVIDIA/nv-rms-client.git", tag = "v0.9.0-rc1" }
 ansi-to-html = "0.2.2"
 

Cargo.toml

@@ -138,6 +138,52 @@ pub struct ServiceAddresses {
     pub nameservers: Vec<IpAddr>,
 }
 
+fn build_dhcp_ntp_servers(
+    nc: &rpc::ManagedHostNetworkConfigResponse,
+    service_addrs: &ServiceAddresses,
+) -> Vec<Ipv4Addr> {
+    // Start with the NTP servers from the service addresses, which is read from carbide-ntp.forge.
+    let mut ntp_servers = service_addrs
+        .ntpservers
+        .iter()
+        .filter_map(|x| match x {
+            IpAddr::V4(x) => Some(*x),
+            _ => None,
+        })
+        .collect::<Vec<Ipv4Addr>>();
+
+    // If the site has configured NTP servers, use them instead.
+    if !nc.ntp_servers.is_empty() {
+        let site_ntp_servers: Vec<Ipv4Addr> = nc.ntp_servers
+        .iter()
+        .filter_map(|s| match IpAddr::from_str(s) {
+            Ok(IpAddr::V4(ip)) => Some(ip),
+            Ok(IpAddr::V6(_)) => {
+                tracing::debug!(
+                    ntp_server = %s,
+                    "IPv6 NTP server from ManagedHostNetworkConfigResponse is ignored for DHCPv4 config"
+                );
+                None
+            }
+            Err(e) => {
+                tracing::debug!(
+                    ntp_server = %s,
+                    error = %e,
+                    "Invalid NTP server IP from ManagedHostNetworkConfigResponse, ignoring"
+                );
+                None
+            }
+        })
+        .collect();
+
+        if !site_ntp_servers.is_empty() {
+            ntp_servers = site_ntp_servers;
+        }
+    }
+
+    ntp_servers
+}
+
 /// How we tell HBN to notice the new file we wrote
 #[derive(Debug)]
 struct PostAction {
@@ -1029,14 +1075,7 @@ async fn update_dhcp_via_grpc(
         })
         .collect::<Vec<Ipv4Addr>>();
 
-    let ntpservers_v4 = service_addrs
-        .ntpservers
-        .iter()
-        .filter_map(|x| match x {
-            IpAddr::V4(x) => Some(*x),
-            _ => None,
-        })
-        .collect::<Vec<Ipv4Addr>>();
+    let ntpservers_v4 = build_dhcp_ntp_servers(network_config, service_addrs);
 
     let pxe_ip_v4 = service_addrs
         .pxe_ips
@@ -1445,14 +1484,7 @@ fn write_dhcp_v4_server_config(
         })
         .collect::<Vec<Ipv4Addr>>();
 
-    let ntpservers_v4 = service_addrs
-        .ntpservers
-        .iter()
-        .filter_map(|x| match x {
-            IpAddr::V4(x) => Some(*x),
-            _ => None,
-        })
-        .collect::<Vec<Ipv4Addr>>();
+    let ntpservers_v4 = build_dhcp_ntp_servers(nc, service_addrs);
 
     let pxe_ip_v4 = service_addrs
         .pxe_ips
@@ -1911,6 +1943,54 @@ mod tests {
         carbide_host_support::init_logging("nico-dpu-agent").unwrap();
     }
 
+    #[test]
+    fn test_build_dhcp_ntp_servers() {
+        let service_addrs = ServiceAddresses {
+            pxe_ips: vec![],
+            ntpservers: vec![IpAddr::from([192, 0, 2, 20])],
+            nameservers: vec![],
+        };
+        let nc = rpc::ManagedHostNetworkConfigResponse {
+            ntp_servers: vec!["198.51.100.1".to_string(), "198.51.100.2".to_string()],
+            ..Default::default()
+        };
+
+        let out = build_dhcp_ntp_servers(&nc, &service_addrs);
+        assert_eq!(
+            out,
+            vec![
+                Ipv4Addr::from([198, 51, 100, 1]),
+                Ipv4Addr::from([198, 51, 100, 2])
+            ]
+        );
+    }
+
+    #[test]
+    fn test_build_dhcp_ntp_servers_fallback() {
+        let service_addrs = ServiceAddresses {
+            pxe_ips: vec![],
+            ntpservers: vec![IpAddr::from([192, 0, 2, 20])],
+            nameservers: vec![],
+        };
+
+        let empty_nc = rpc::ManagedHostNetworkConfigResponse::default();
+
+        assert_eq!(
+            build_dhcp_ntp_servers(&empty_nc, &service_addrs),
+            vec![Ipv4Addr::from([192, 0, 2, 20])]
+        );
+
+        let invalid_nc = rpc::ManagedHostNetworkConfigResponse {
+            ntp_servers: vec!["not-an-ip".to_string(), "2001:db8::1".to_string()],
+            ..Default::default()
+        };
+
+        assert_eq!(
+            build_dhcp_ntp_servers(&invalid_nc, &service_addrs),
+            vec![Ipv4Addr::from([192, 0, 2, 20])]
+        );
+    }
+
     #[test]
     fn test_hostname() -> Result<(), Box<dyn std::error::Error>> {
         let syscall_h = super::hostname()?;
@@ -2935,6 +3015,7 @@ mod tests {
 
             // yes it's in there twice I dunno either
             dhcp_servers: vec!["10.217.5.197".to_string(), "10.217.5.197".to_string()],
+            ntp_servers: vec![],
             vni_device: "vxlan48".to_string(),
 
             managed_host_config: Some(netconf),
@@ -3421,6 +3502,7 @@ mod tests {
 
             // yes it's in there twice I dunno either
             dhcp_servers: vec!["10.217.5.197".to_string(), "10.217.5.197".to_string()],
+            ntp_servers: vec![],
             vni_device: "vxlan48".to_string(),
 
             managed_host_config: Some(netconf),
@@ -3616,6 +3698,7 @@ mod tests {
             routing_profile: None,
             traffic_intercept_config: None,
             dhcp_servers: vec![],
+            ntp_servers: vec![],
             vni_device: "vxlan48".to_string(),
             managed_host_config: Some(netconf),
             managed_host_config_version: "V1-T1".to_string(),

crates/agent/src/ethernet_virtualization.rs

@@ -885,6 +885,7 @@ async fn handle_netconf(AxumState(state): AxumState<Arc<Mutex<State>>>) -> impl
         }),
 
         dhcp_servers: vec!["127.0.0.1".to_string()],
+        ntp_servers: vec![],
         vni_device: "".to_string(),
 
         managed_host_config: Some(rpc::forge::ManagedHostNetworkConfig {

crates/agent/src/tests/full.rs

@@ -20,6 +20,7 @@ applicable.
 | `ib_config` | `Option<IBFabricConfig>` | — | InfiniBand fabric configuration (see [IBFabricConfig](#ibfabricconfig)). |
 | `asn` | `u32` | **required** | Autonomous System Number, fixed per environment. Used by nico-dpu-agent for `frr.conf` BGP routing. |
 | `dhcp_servers` | `Vec<Ipv4Addr>` | `[]` | DHCP server addresses announced to DPUs during network provisioning. |
+| `ntp_servers` | `Vec<Ipv4Addr>` | `[]` | Site-level NTP server IPs used for BMC time configuration and DHCP NTP Server configuration. |
 | `route_servers` | `Vec<String>` | `[]` | Route server IPs for L2VPN Ethernet Virtual network support. |
 | `enable_route_servers` | `bool` | `false` | Enables route server injection into DPU FRR configs for L2VPN. |
 | `deny_prefixes` | `Vec<Ipv4Network>` | `[]` | IPv4 CIDR prefixes that tenant instances are blocked from reaching. Generates iptables DROP rules and nvue ACL policies. |

crates/api-core/src/cfg/README.md

@@ -128,6 +128,10 @@ pub struct CarbideConfig {
     #[serde(default)]
     pub dhcp_servers: Vec<Ipv4Addr>,
 
+    /// NTP server IP addresses for the site.
+    #[serde(default)]
+    pub ntp_servers: Vec<Ipv4Addr>,
+
     /// Route server IP addresses for L2VPN (Ethernet
     /// Virtual) network support on DPUs.
     #[serde(default)]
@@ -2863,6 +2867,10 @@ mod tests {
             config.dhcp_servers,
             vec![Ipv4Addr::new(1, 2, 3, 4), Ipv4Addr::new(5, 6, 7, 8)]
         );
+        assert_eq!(
+            config.ntp_servers,
+            vec![Ipv4Addr::new(10, 20, 30, 40), Ipv4Addr::new(50, 60, 70, 80)]
+        );
         assert_eq!(config.vpc_peering_policy, Some(VpcPeeringPolicy::Exclusive));
         assert_eq!(
             config.vpc_peering_policy_on_existing,

crates/api-core/src/cfg/file.rs

@@ -4,6 +4,7 @@ database_url = "postgres://a:b@postgresql"
 max_database_connections = 1222
 asn = 123
 dhcp_servers = ["1.2.3.4", "5.6.7.8"]
+ntp_servers = ["10.20.30.40", "50.60.70.80"]
 route_servers = ["9.10.11.12"]
 rapid_iterations = false
 initial_domain_name = "forge.local"

crates/api-core/src/cfg/test_data/full_config.toml

@@ -52,6 +52,7 @@ async fn handle_overlay_from_dpa(
     dpa_if: &mut DpaInterface,
     macaddr: MacAddress,
     desired_addr: IpAddr,
+    ntp_servers: &[Ipv4Addr],
 ) -> Result<Option<Response<rpc::DhcpRecord>>, CarbideError> {
     let IpAddr::V4(ip_v4_addr) = desired_addr else {
         return Err(CarbideError::internal(
@@ -80,6 +81,7 @@ async fn handle_overlay_from_dpa(
         mtu: SPX_MTU,
         fqdn: String::new(),
         prefix,
+        ntp_servers: ntp_servers.iter().map(ToString::to_string).collect(),
     })))
 }
 
@@ -90,6 +92,7 @@ async fn handle_underlay_from_dpa(
     dpa_if: &mut DpaInterface,
     macaddr: MacAddress,
     relay_address: String,
+    ntp_servers: &[Ipv4Addr],
 ) -> Result<Option<Response<rpc::DhcpRecord>>, CarbideError> {
     // The relay address and the mac address should differ only in bit 0
     let relay_addr = Ipv4Addr::from_str(&relay_address)?;
@@ -119,6 +122,7 @@ async fn handle_underlay_from_dpa(
         mtu: SPX_MTU,
         fqdn: String::new(),
         prefix,
+        ntp_servers: ntp_servers.iter().map(ToString::to_string).collect(),
     })))
 }
 
@@ -156,10 +160,24 @@ async fn handle_dhcp_from_dpa(
     let mut dpa_if = dpa_ifs.remove(0);
 
     if let Some(addr) = desired_address {
-        return handle_overlay_from_dpa(txn, &mut dpa_if, macaddr, addr).await;
+        return handle_overlay_from_dpa(
+            txn,
+            &mut dpa_if,
+            macaddr,
+            addr,
+            &api.runtime_config.ntp_servers,
+        )
+        .await;
     }
 
-    handle_underlay_from_dpa(txn, &mut dpa_if, macaddr, relay_address).await
+    handle_underlay_from_dpa(
+        txn,
+        &mut dpa_if,
+        macaddr,
+        relay_address,
+        &api.runtime_config.ntp_servers,
+    )
+    .await
 }
 
 pub async fn discover_dhcp(
@@ -437,7 +455,7 @@ pub async fn discover_dhcp(
 
     let mut txn = api.txn_begin().await?;
 
-    let record: rpc::DhcpRecord = db::dhcp_record::find_by_mac_address(
+    let mut record: rpc::DhcpRecord = db::dhcp_record::find_by_mac_address(
         &mut txn,
         &parsed_mac,
         &machine_interface.segment_id,
@@ -447,5 +465,13 @@ pub async fn discover_dhcp(
     .into();
 
     txn.commit().await?;
+
+    record.ntp_servers = api
+        .runtime_config
+        .ntp_servers
+        .iter()
+        .map(ToString::to_string)
+        .collect();
+
     Ok(Response::new(record))
 }

crates/api-core/src/dhcp/discover.rs

@@ -595,6 +595,12 @@ pub(crate) async fn get_managed_host_network_config_inner(
             .map(|addr| addr.to_string())
             .collect(),
         route_servers,
+        ntp_servers: api
+            .runtime_config
+            .ntp_servers
+            .iter()
+            .map(|addr| addr.to_string())
+            .collect(),
         // TODO: Automatically add the prefix(es?) from the IPv4 loopback
         // pool to deny_prefixes. The database stores the pool in an
         // exploded representation, so we either need to reconstruct the

crates/api-core/src/handlers/dpu.rs

@@ -1463,6 +1463,7 @@ async fn initialize_and_start_controllers<'a>(
         Some(upload_limiter),
         Some(api_service.credential_manager.clone()),
         work_lock_manager_handle.clone(),
+        carbide_config.ntp_servers.clone(),
     )
     .start(join_set, cancel_token.clone())?;