nova05: add cross-site routes/GWs and annotations

For all nova05epsilon's NADs, NetConfig subnets, host-level NNCP routes
add missing cross-site DCN routes, gateways, and
storagemgmt.endpoint_annotation.

Previsourely those were tested in ci-framework jobs overlays but the
base DT example was missing those CHANGEME templates.

Signed-off-by: Bohdan Dobrelia <bdobreli@redhat.com>

Author: bogdando

examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml

@@ -30,7 +30,11 @@ data:
                   "type": "whereabouts",
                   "range": "CHANGEME_SNO_CTLPLANE_CIDR",
                   "range_start": "CHANGEME_SNO_CTLPLANE_NAD_START",
-                  "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END"
+                  "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END",
+                  "routes": [
+                    { "dst": "CHANGEME_EDPM_CTLPLANE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" },
+                    { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" }
+                  ]
                 }
               }, {
                 "type": "tuning",
@@ -50,6 +54,13 @@ data:
               gateway: CHANGEME_SNO_CTLPLANE_GW
               name: subnet1
               vlan: CHANGEME_SNO_CTLPLANE_VLAN
+              routes:
+                  - destination: CHANGEME_EDPM_CTLPLANE_CIDR
+                    nexthop: CHANGEME_SNO_CTLPLANE_GW
+                  # Pods with only ctlplane attached (e.g. tempest) need this
+                  # to reach Ceph RGW on the remote storage network.
+                  - destination: CHANGEME_EDPM_STORAGE_CIDR
+                    nexthop: CHANGEME_SNO_CTLPLANE_GW
             # CHANGEME: site4 EDPM computes -- replace CIDRs/gateway/VLAN
             - allocationRanges:
                   - end: CHANGEME_EDPM_CTLPLANE_END
@@ -58,6 +69,9 @@ data:
               gateway: CHANGEME_EDPM_CTLPLANE_GW
               name: subnet2
               vlan: CHANGEME_EDPM_CTLPLANE_VLAN
+              routes:
+                  - destination: CHANGEME_SNO_CTLPLANE_CIDR
+                    nexthop: CHANGEME_EDPM_CTLPLANE_GW
     datacentre:
         net-attach-def: |
             {
@@ -126,7 +140,10 @@ data:
                   "type": "whereabouts",
                   "range": "CHANGEME_SNO_INTAPI_CIDR",
                   "range_start": "CHANGEME_SNO_INTAPI_NAD_START",
-                  "range_end": "CHANGEME_SNO_INTAPI_NAD_END"
+                  "range_end": "CHANGEME_SNO_INTAPI_NAD_END",
+                  "routes": [
+                    { "dst": "CHANGEME_EDPM_INTAPI_CIDR", "gw": "CHANGEME_SNO_INTAPI_GW" }
+                  ]
                 }
               }, {
                 "type": "tuning",
@@ -141,15 +158,23 @@ data:
                   - end: CHANGEME_SNO_INTAPI_END
                     start: CHANGEME_SNO_INTAPI_START
               cidr: CHANGEME_SNO_INTAPI_CIDR
+              gateway: CHANGEME_SNO_INTAPI_GW
               name: subnet1
               vlan: CHANGEME_SNO_INTAPI_VLAN
+              routes:
+                  - destination: CHANGEME_EDPM_INTAPI_CIDR
+                    nexthop: CHANGEME_SNO_INTAPI_GW
             # CHANGEME: site4 internalapi subnet
             - allocationRanges:
                   - end: CHANGEME_EDPM_INTAPI_END
                     start: CHANGEME_EDPM_INTAPI_START
               cidr: CHANGEME_EDPM_INTAPI_CIDR
+              gateway: CHANGEME_EDPM_INTAPI_GW
               name: subnet2
               vlan: CHANGEME_EDPM_INTAPI_VLAN
+              routes:
+                  - destination: CHANGEME_SNO_INTAPI_CIDR
+                    nexthop: CHANGEME_EDPM_INTAPI_GW
         vlan: CHANGEME_SNO_INTAPI_VLAN
     lbServiceType: LoadBalancer
     # SNO: single OCP node only
@@ -183,7 +208,24 @@ data:
             metallb.universe.tf/address-pool: internalapi
             metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_CELL1_IP
     routes:
-        config: []
+        config:
+            # CHANGEME: host-level routes for cross-site reachability (EDPM subnet2)
+            - destination: CHANGEME_EDPM_CTLPLANE_CIDR
+              next-hop-address: CHANGEME_SNO_CTLPLANE_GW
+              next-hop-interface: ospbr
+              table-id: 254
+            - destination: CHANGEME_EDPM_INTAPI_CIDR
+              next-hop-address: CHANGEME_SNO_INTAPI_GW
+              next-hop-interface: internalapi
+              table-id: 254
+            - destination: CHANGEME_EDPM_STORAGE_CIDR
+              next-hop-address: CHANGEME_SNO_STORAGE_GW
+              next-hop-interface: storage
+              table-id: 254
+            - destination: CHANGEME_EDPM_TENANT_CIDR
+              next-hop-address: CHANGEME_SNO_TENANT_GW
+              next-hop-interface: tenant
+              table-id: 254
     storage:
         base_iface: CHANGEME_SNO_IFACE
         dnsDomain: storage.example.com
@@ -206,7 +248,10 @@ data:
                   "type": "whereabouts",
                   "range": "CHANGEME_SNO_STORAGE_CIDR",
                   "range_start": "CHANGEME_SNO_STORAGE_NAD_START",
-                  "range_end": "CHANGEME_SNO_STORAGE_NAD_END"
+                  "range_end": "CHANGEME_SNO_STORAGE_NAD_END",
+                  "routes": [
+                    { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_STORAGE_GW" }
+                  ]
                 }
               }, {
                 "type": "tuning",
@@ -221,20 +266,32 @@ data:
                   - end: CHANGEME_SNO_STORAGE_END
                     start: CHANGEME_SNO_STORAGE_START
               cidr: CHANGEME_SNO_STORAGE_CIDR
+              gateway: CHANGEME_SNO_STORAGE_GW
               name: subnet1
               vlan: CHANGEME_SNO_STORAGE_VLAN
+              routes:
+                  - destination: CHANGEME_EDPM_STORAGE_CIDR
+                    nexthop: CHANGEME_SNO_STORAGE_GW
             # CHANGEME: site4 storage subnet
             - allocationRanges:
                   - end: CHANGEME_EDPM_STORAGE_END
                     start: CHANGEME_EDPM_STORAGE_START
               cidr: CHANGEME_EDPM_STORAGE_CIDR
+              gateway: CHANGEME_EDPM_STORAGE_GW
               name: subnet2
               vlan: CHANGEME_EDPM_STORAGE_VLAN
+              routes:
+                  - destination: CHANGEME_SNO_STORAGE_CIDR
+                    nexthop: CHANGEME_EDPM_STORAGE_GW
         vlan: CHANGEME_SNO_STORAGE_VLAN
     storageClass: lvms-local-storage
     storagemgmt:
         base_iface: CHANGEME_SNO_IFACE
         dnsDomain: storagemgmt.example.com
+        endpoint_annotations:
+            metallb.universe.tf/address-pool: storagemgmt
+            metallb.universe.tf/allow-shared-ip: storagemgmt
+            metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_STGMGMT_LB_IP
         iface: storagemgmt
         lb_addresses:
             - CHANGEME_SNO_STGMGMT_LB_START-CHANGEME_SNO_STGMGMT_LB_END
@@ -250,7 +307,10 @@ data:
                   "type": "whereabouts",
                   "range": "CHANGEME_SNO_STGMGMT_CIDR",
                   "range_start": "CHANGEME_SNO_STGMGMT_NAD_START",
-                  "range_end": "CHANGEME_SNO_STGMGMT_NAD_END"
+                  "range_end": "CHANGEME_SNO_STGMGMT_NAD_END",
+                  "routes": [
+                    { "dst": "CHANGEME_EDPM_STGMGMT_CIDR", "gw": "CHANGEME_SNO_STGMGMT_GW" }
+                  ]
                 }
               }, {
                 "type": "tuning",
@@ -265,15 +325,23 @@ data:
                   - end: CHANGEME_SNO_STGMGMT_END
                     start: CHANGEME_SNO_STGMGMT_START
               cidr: CHANGEME_SNO_STGMGMT_CIDR
+              gateway: CHANGEME_SNO_STGMGMT_GW
               name: subnet1
               vlan: CHANGEME_SNO_STGMGMT_VLAN
+              routes:
+                  - destination: CHANGEME_EDPM_STGMGMT_CIDR
+                    nexthop: CHANGEME_SNO_STGMGMT_GW
             # CHANGEME: site4 storagemgmt subnet
             - allocationRanges:
                   - end: CHANGEME_EDPM_STGMGMT_END
                     start: CHANGEME_EDPM_STGMGMT_START
               cidr: CHANGEME_EDPM_STGMGMT_CIDR
+              gateway: CHANGEME_EDPM_STGMGMT_GW
               name: subnet2
               vlan: CHANGEME_EDPM_STGMGMT_VLAN
+              routes:
+                  - destination: CHANGEME_SNO_STGMGMT_CIDR
+                    nexthop: CHANGEME_EDPM_STGMGMT_GW
         vlan: CHANGEME_SNO_STGMGMT_VLAN
     tenant:
         base_iface: CHANGEME_SNO_IFACE
@@ -297,7 +365,10 @@ data:
                   "type": "whereabouts",
                   "range": "CHANGEME_SNO_TENANT_CIDR",
                   "range_start": "CHANGEME_SNO_TENANT_NAD_START",
-                  "range_end": "CHANGEME_SNO_TENANT_NAD_END"
+                  "range_end": "CHANGEME_SNO_TENANT_NAD_END",
+                  "routes": [
+                    { "dst": "CHANGEME_EDPM_TENANT_CIDR", "gw": "CHANGEME_SNO_TENANT_GW" }
+                  ]
                 }
               }, {
                 "type": "tuning",
@@ -312,15 +383,23 @@ data:
                   - end: CHANGEME_SNO_TENANT_END
                     start: CHANGEME_SNO_TENANT_START
               cidr: CHANGEME_SNO_TENANT_CIDR
+              gateway: CHANGEME_SNO_TENANT_GW
               name: subnet1
               vlan: CHANGEME_SNO_TENANT_VLAN
+              routes:
+                  - destination: CHANGEME_EDPM_TENANT_CIDR
+                    nexthop: CHANGEME_SNO_TENANT_GW
             # CHANGEME: site4 tenant subnet
             - allocationRanges:
                   - end: CHANGEME_EDPM_TENANT_END
                     start: CHANGEME_EDPM_TENANT_START
               cidr: CHANGEME_EDPM_TENANT_CIDR
+              gateway: CHANGEME_EDPM_TENANT_GW
               name: subnet2
               vlan: CHANGEME_EDPM_TENANT_VLAN
+              routes:
+                  - destination: CHANGEME_SNO_TENANT_CIDR
+                    nexthop: CHANGEME_EDPM_TENANT_GW
         vlan: CHANGEME_SNO_TENANT_VLAN
 
 kind: ConfigMap