Skip to content

Commit 62637f7

Browse files
jgallagherjgallagher
authored
Add service zone NAT entries to bootstore network config (#10219)
Followup to #10218: actually fill in the new service zone NAT entries field we added with real values taken from either the RSS service plan (during rack setup) or the blueprint (in Nexus). Closes #10168.
1 parent 831e013 commit 62637f7

11 files changed

Lines changed: 235 additions & 38 deletions

File tree

‎nexus/db-queries/src/db/datastore/bootstore.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use diesel::prelude::*;
77
use nexus_db_errors::{ErrorHandler, public_error_from_diesel};
88
use nexus_db_model::{BootstoreConfig, BootstoreKeys};
99
use omicron_common::api::external::{CreateResult, LookupResult};
10+
use sled_agent_types::rack_init::rack_init_bootstore_generation;
1011

1112
impl DataStore {
1213
pub async fn bump_bootstore_generation(
@@ -22,7 +23,7 @@ impl DataStore {
2223
let bks = diesel::insert_into(dsl::bootstore_keys)
2324
.values(BootstoreKeys {
2425
key: key.clone(),
25-
generation: 2, // RSS starts with a generation of 1
26+
generation: rack_init_bootstore_generation::NEXUS_INITIAL,
2627
})
2728
.on_conflict(bootstore_keys::key)
2829
.do_update()

‎nexus/src/app/background/init.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,7 @@ impl BackgroundTasksInitializer {
728728
task_impl: Box::new(SwitchPortSettingsManager::new(
729729
datastore.clone(),
730730
resolver.clone(),
731+
rx_blueprint.clone(),
731732
)),
732733
opctx: opctx.child(BTreeMap::new()),
733734
watchers: vec![],

‎nexus/src/app/background/tasks/sync_switch_configuration.rs‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
//! to relevant management daemons (dendrite, mgd, sled-agent, etc.)
77
88
use crate::app::{
9-
background::tasks::networking::{
10-
api_to_dpd_port_settings, build_mgd_clients,
9+
background::{
10+
LoadedTargetBlueprint,
11+
tasks::networking::{api_to_dpd_port_settings, build_mgd_clients},
1112
},
1213
dpd_clients, switch_zone_address_mappings,
1314
};
@@ -20,6 +21,7 @@ use nexus_db_model::{
2021
AddressLotBlock, BgpConfig, BootstoreConfig, INFRA_LOT, LoopbackAddress,
2122
NETWORK_KEY, SwitchLinkSpeed,
2223
};
24+
use tokio::sync::watch;
2325
use uuid::Uuid;
2426

2527
use crate::app::background::BackgroundTask;
@@ -131,11 +133,16 @@ impl Default for AddStaticRouteRequest {
131133
pub struct SwitchPortSettingsManager {
132134
datastore: Arc<DataStore>,
133135
resolver: Resolver,
136+
rx_blueprint: watch::Receiver<Option<LoadedTargetBlueprint>>,
134137
}
135138

136139
impl SwitchPortSettingsManager {
137-
pub fn new(datastore: Arc<DataStore>, resolver: Resolver) -> Self {
138-
Self { datastore, resolver }
140+
pub fn new(
141+
datastore: Arc<DataStore>,
142+
resolver: Resolver,
143+
rx_blueprint: watch::Receiver<Option<LoadedTargetBlueprint>>,
144+
) -> Self {
145+
Self { datastore, resolver, rx_blueprint }
139146
}
140147

141148
async fn switch_ports(
@@ -1301,6 +1308,28 @@ impl BackgroundTask for SwitchPortSettingsManager {
13011308
}
13021309
};
13031310

1311+
let service_zone_nat_entries = match self
1312+
.rx_blueprint
1313+
.borrow_and_update()
1314+
.clone()
1315+
.map(|bp| bp.blueprint.to_service_zone_nat_entries())
1316+
{
1317+
Some(Ok(entries)) => entries,
1318+
Some(Err(err)) => {
1319+
error!(
1320+
log,
1321+
"cannot construct service zone NAT entries \
1322+
from blueprint";
1323+
InlineErrorChain::new(&err),
1324+
);
1325+
continue;
1326+
}
1327+
None => {
1328+
warn!(log, "blueprint not yet loaded - skipping sync");
1329+
continue;
1330+
}
1331+
};
1332+
13041333
let desired_config = SystemNetworkingConfig {
13051334
rack_network_config: RackNetworkConfig {
13061335
rack_subnet: subnet,
@@ -1310,9 +1339,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
13101339
bgp,
13111340
bfd,
13121341
},
1313-
// TODO-correctness We need to fill this in based on the
1314-
// current blueprint.
1315-
service_zone_nat_entries: None,
1342+
service_zone_nat_entries: Some(service_zone_nat_entries),
13161343
};
13171344

13181345
// bootstore_needs_update is a boolean value that determines

‎nexus/types/src/deployment.rs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use ipnet::IpAdd;
3333
use omicron_common::address::Ipv6Subnet;
3434
use omicron_common::address::SLED_PREFIX;
3535
use omicron_common::address::SLED_RESERVED_ADDRESSES;
36+
use omicron_common::address::get_sled_address;
3637
use omicron_common::api::external::ByteCount;
3738
use omicron_common::api::external::Generation;
3839
use omicron_common::api::external::TufArtifactMeta;
@@ -54,6 +55,10 @@ use omicron_uuid_kinds::ZpoolUuid;
5455
use schemars::JsonSchema;
5556
use serde::Deserialize;
5657
use serde::Serialize;
58+
use sled_agent_types::system_networking::ServiceZoneNatEntries;
59+
use sled_agent_types::system_networking::ServiceZoneNatEntriesError;
60+
use sled_agent_types::system_networking::ServiceZoneNatEntry;
61+
use sled_agent_types::system_networking::ServiceZoneNatKind;
5762
use sled_agent_types_versions::latest::inventory::HostPhase2DesiredContents;
5863
use sled_agent_types_versions::latest::inventory::HostPhase2DesiredSlots;
5964
use sled_agent_types_versions::latest::inventory::OmicronSingleMeasurement;
@@ -313,6 +318,79 @@ impl Blueprint {
313318
}
314319
}
315320

321+
/// Construct a [`ServiceZoneNatEntries`] containing all NAT entries for
322+
/// relevant in-service zones in this blueprint.
323+
///
324+
/// # Errors
325+
///
326+
/// This method will fail if the blueprint contains overlapping NAT entries
327+
/// (never expected) or has no in-service zones of the types required by
328+
/// `ServiceZoneNatEntries` (boundary NTP, external DNS, and Nexus). Real
329+
/// blueprints should always have at least one zone of this type, but many
330+
/// test blueprints will not.
331+
pub fn to_service_zone_nat_entries(
332+
&self,
333+
) -> Result<ServiceZoneNatEntries, ServiceZoneNatEntriesError> {
334+
let entries = self
335+
.in_service_zones()
336+
.filter_map(|(sled_id, zone_config)| {
337+
let (nic_mac, vni, kind) = match &zone_config.zone_type {
338+
BlueprintZoneType::BoundaryNtp(ntp) => (
339+
ntp.nic.mac,
340+
ntp.nic.vni,
341+
ServiceZoneNatKind::BoundaryNtp {
342+
snat_cfg: ntp.external_ip.snat_cfg,
343+
},
344+
),
345+
BlueprintZoneType::ExternalDns(dns) => (
346+
dns.nic.mac,
347+
dns.nic.vni,
348+
ServiceZoneNatKind::ExternalDns {
349+
external_ip: dns.dns_address.addr.ip(),
350+
},
351+
),
352+
BlueprintZoneType::Nexus(nexus) => (
353+
nexus.nic.mac,
354+
nexus.nic.vni,
355+
ServiceZoneNatKind::Nexus {
356+
external_ip: nexus.external_ip.ip,
357+
},
358+
),
359+
360+
// None of these zone types have external NAT.
361+
BlueprintZoneType::Clickhouse(_)
362+
| BlueprintZoneType::ClickhouseKeeper(_)
363+
| BlueprintZoneType::ClickhouseServer(_)
364+
| BlueprintZoneType::CockroachDb(_)
365+
| BlueprintZoneType::Crucible(_)
366+
| BlueprintZoneType::CruciblePantry(_)
367+
| BlueprintZoneType::InternalDns(_)
368+
| BlueprintZoneType::InternalNtp(_)
369+
| BlueprintZoneType::Oximeter(_) => return None,
370+
};
371+
372+
// in_service_zones() iterates over `self.sleds`, so it can only
373+
// give us `sled_id`s that exist there. It's safe for us to
374+
// unwrap here.
375+
let sled_subnet = self
376+
.sleds
377+
.get(&sled_id)
378+
.expect("sled must exist if we have in-service zones")
379+
.subnet;
380+
381+
Some(ServiceZoneNatEntry {
382+
zone_id: zone_config.id,
383+
sled_underlay_ip: *get_sled_address(sled_subnet).ip(),
384+
nic_mac,
385+
vni,
386+
kind,
387+
})
388+
})
389+
.collect::<IdOrdMap<_>>();
390+
391+
entries.try_into()
392+
}
393+
316394
/// Iterate over the in-service [`BlueprintZoneConfig`] instances in the
317395
/// blueprint, along with the associated sled id.
318396
pub fn in_service_zones(

‎openapi/bootstrap-agent-lockstep.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1383,7 +1383,7 @@
13831383
"status": {
13841384
"type": "string",
13851385
"enum": [
1386-
"network_config_update"
1386+
"initial_network_config_update"
13871387
]
13881388
}
13891389
},
@@ -1405,6 +1405,20 @@
14051405
"status"
14061406
]
14071407
},
1408+
{
1409+
"type": "object",
1410+
"properties": {
1411+
"status": {
1412+
"type": "string",
1413+
"enum": [
1414+
"final_network_config_update"
1415+
]
1416+
}
1417+
},
1418+
"required": [
1419+
"status"
1420+
]
1421+
},
14081422
{
14091423
"type": "object",
14101424
"properties": {

‎openapi/wicketd.json‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3901,7 +3901,7 @@
39013901
"maximum": 9000
39023902
},
39033903
"RssStep": {
3904-
"description": "Steps we go through during initial rack setup. Keep this list in order that they happen.\n\n<details><summary>JSON schema</summary>\n\n```json { \"description\": \"Steps we go through during initial rack setup. Keep this list in order that they happen.\", \"oneOf\": [ { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"requested\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"starting\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"load_existing_plan\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"create_sled_plan\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"init_trust_quorum\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"network_config_update\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"sled_init\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"init_dns\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"configure_dns\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"init_ntp\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"wait_for_time_sync\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"wait_for_database\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"cluster_init\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"zones_init\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"nexus_handoff\" ] } } } ] } ``` </details>",
3904+
"description": "Steps we go through during initial rack setup. Keep this list in order that they happen.\n\n<details><summary>JSON schema</summary>\n\n```json { \"description\": \"Steps we go through during initial rack setup. Keep this list in order that they happen.\", \"oneOf\": [ { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"requested\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"starting\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"load_existing_plan\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"create_sled_plan\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"init_trust_quorum\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"initial_network_config_update\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"sled_init\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"final_network_config_update\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"init_dns\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"configure_dns\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"init_ntp\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"wait_for_time_sync\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"wait_for_database\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"cluster_init\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"zones_init\" ] } } }, { \"type\": \"object\", \"required\": [ \"status\" ], \"properties\": { \"status\": { \"type\": \"string\", \"enum\": [ \"nexus_handoff\" ] } } } ] } ``` </details>",
39053905
"oneOf": [
39063906
{
39073907
"type": "object",
@@ -3979,7 +3979,7 @@
39793979
"status": {
39803980
"type": "string",
39813981
"enum": [
3982-
"network_config_update"
3982+
"initial_network_config_update"
39833983
]
39843984
}
39853985
},
@@ -4001,6 +4001,20 @@
40014001
"status"
40024002
]
40034003
},
4004+
{
4005+
"type": "object",
4006+
"properties": {
4007+
"status": {
4008+
"type": "string",
4009+
"enum": [
4010+
"final_network_config_update"
4011+
]
4012+
}
4013+
},
4014+
"required": [
4015+
"status"
4016+
]
4017+
},
40044018
{
40054019
"type": "object",
40064020
"properties": {

‎sled-agent/bootstrap-agent-lockstep-types/src/lib.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,8 +295,9 @@ pub enum RssStep {
295295
LoadExistingPlan,
296296
CreateSledPlan,
297297
InitTrustQuorum,
298-
NetworkConfigUpdate,
298+
InitialNetworkConfigUpdate,
299299
SledInit,
300+
FinalNetworkConfigUpdate,
300301
InitDns,
301302
ConfigureDns,
302303
InitNtp,

0 commit comments

Comments
 (0)