feat: pin BGP source address via numbered underlay links

privateip · privateip · commit 15fb427d2e36 · 2026-06-22T21:38:31.000-04:00
Replace BGP unnumbered (link-local) underlay peering with numbered
IPv6 /64 subnets between workers and transit routers. Add
BGP_LOCAL_ADDRESS env var to the galactic-router overlay DaemonSet
so GoBGP pins the TCP source address to the node SRv6 loopback.

Underlay: configure numbered IPv6 links and route-maps to set source
address on FRR BGP advertisements (SRv6 SID/forwarding prefixes).

GoBGP runtime: accept localAddress in NewRuntimeFactory, propagate
to peerFromDesired, set Transport.LocalAddress on every peer.

Docs: update containerlab README to reflect numbered links.
diff --git a/cmd/galactic-router/main.go b/cmd/galactic-router/main.go
@@ -50,10 +50,12 @@ func main() {
 		bgpListenPort = int32(p)
 	}
 
+	bgpLocalAddr := os.Getenv("BGP_LOCAL_ADDRESS")
+
 	var factory galacticruntime.RuntimeFactory
 	switch routerRole {
 	case "tenant":
-		factory = gobgp.NewRuntimeFactory(bgpListenPort)
+		factory = gobgp.NewRuntimeFactory(bgpListenPort, bgpLocalAddr)
 	case "fabric":
 		factory = frr.NewRuntimeFactory()
 	default:
diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md
@@ -2,7 +2,7 @@
 
 Three Kind clusters (dfw, iad, sjc) connected over an IPv6 SRv6 transit mesh. Each cluster
 runs FRR as a node routing daemon (hostNetwork DaemonSet) to peer with the transit layer via
-BGP unnumbered. galactic-router runs alongside FRR on the workers to distribute EVPN routes
+eBGP over numbered IPv6 links. galactic-router runs alongside FRR on the workers to distribute EVPN routes
 over iBGP to the route reflector on iad-rr.
 
 ## Topology
@@ -36,10 +36,10 @@ over iBGP to the route reflector on iad-rr.
 ### BGP design
 
 ```
-AS 65000 (dfw-underlay / FRR)          ──eBGP unnumbered──  tr1 (AS 65100)
-AS 65000 (iad-underlay / FRR)          ──eBGP unnumbered──  tr3:eth5 (AS 65100)
-AS 65000 (iad-rr-underlay / FRR)       ──eBGP unnumbered──  tr3:eth4 (AS 65100)
-AS 65000 (sjc-underlay / FRR)          ──eBGP unnumbered──  tr2 (AS 65100)
+AS 65000 (dfw-underlay / FRR)          ──eBGP──  tr1 (AS 65100)
+AS 65000 (iad-underlay / FRR)          ──eBGP──  tr3:eth5 (AS 65100)
+AS 65000 (iad-rr-underlay / FRR)       ──eBGP──  tr3:eth4 (AS 65100)
+AS 65000 (sjc-underlay / FRR)          ──eBGP──  tr2 (AS 65100)
 
 AS 65000 (dfw-overlay / galactic-router)  ──iBGP──  iad-rr (AS 65000 RR)
 AS 65000 (iad-overlay / galactic-router)  ──iBGP──  iad-rr (AS 65000 RR)
@@ -48,7 +48,7 @@ AS 65000 (sjc-overlay / galactic-router)  ──iBGP──  iad-rr (AS 65000 RR)
 
 - All clusters use a single AS (65000) for both the FRR underlay and the galactic-router overlay.
 - The transit mesh carries IPv6 unicast (SRv6 locator prefixes and loopbacks) via iBGP within AS 65100.
-- FRR PE nodes originate their SRv6 forwarding prefix (`2001:db8:ffXX::/48`) and SRv6 SID block (`fc00:0:X::/48`) toward the transit layer via eBGP unnumbered.
+- FRR PE nodes originate their SRv6 forwarding prefix (`2001:db8:ffXX::/48`) and SRv6 SID block (`fc00:0:X::/48`) toward the transit layer via eBGP over numbered IPv6 links.
 - `allowas-in 1` is configured on all cluster FRR instances so each site accepts prefixes that carry AS 65000 in the path — necessary because the transit reflects routes from one AS 65000 site to another.
 - galactic-router instances on dfw/iad/sjc workers peer with iad-worker-rr over iBGP (AS 65000) for `l2vpn-evpn` routes. GoBGP runs with outbound-only mode (`listenPort=-1`); all BGP sessions are initiated outbound.
 
@@ -74,14 +74,14 @@ AS 65000 (sjc-overlay / galactic-router)  ──iBGP──  iad-rr (AS 65000 RR)
 | tr2–tr4 | 2001:db8:0:24::/64  |
 | tr3–tr4 | 2001:db8:0:34::/64  |
 
-### Worker–TR links (BGP unnumbered, link-local only)
+### Worker–TR links (numbered, eBGP)
 
-| Link                   | TR interface |
-|------------------------|--------------|
-| dfw-worker – tr1       | eth1         |
-| sjc-worker – tr2       | eth1         |
-| iad-worker – tr3       | eth5         |
-| iad-worker-rr – tr3    | eth4         |
+| Link                   | Subnet              | TR address     | Worker address   |
+|------------------------|---------------------|----------------|------------------|
+| dfw-worker – tr1       | 2001:db8:1:10::/64  | 2001:db8:1:10::1 | 2001:db8:1:10::2 |
+| sjc-worker – tr2       | 2001:db8:1:20::/64  | 2001:db8:1:20::1 | 2001:db8:1:20::2 |
+| iad-worker – tr3       | 2001:db8:1:30::/64  | 2001:db8:1:30::1 | 2001:db8:1:30::2 |
+| iad-worker-rr – tr3    | 2001:db8:1:31::/64  | 2001:db8:1:31::1 | 2001:db8:1:31::2 |
 
 ### Cluster SRv6 addressing
 
@@ -237,8 +237,7 @@ docker exec sjc-control-plane kubectl get bgprouters -A
 - All three Kind clusters use `disableDefaultCNI: true`. Cilium is installed by the
   `kindest/node:galactic` bootstrap script. cert-manager and Multus are only installed
   on iad and sjc.
-- Worker–TR links use BGP unnumbered (IPv6 link-local only). No numbered addresses are
-  configured on worker data-plane interfaces.
+- Worker–TR links use numbered IPv6 subnets (/64) with eBGP peering.
 - Cilium's iptables rules block BGP by default; the bootstrap script inserts
   `ip6tables -I INPUT` rules for TCP/179 before Cilium starts on each worker.
 - iad-worker-rr peers with tr3 as AS 65000, the same AS used by all three clusters.
diff --git a/deploy/containerlab/node_files/tr1/frr.conf b/deploy/containerlab/node_files/tr1/frr.conf
@@ -7,7 +7,8 @@ interface lo
  ipv6 address fc00:0:1::1/128
 !
 interface eth1
- description iad-worker-facing
+ description dfw-worker-facing
+ ipv6 address 2001:db8:1:10::1/64
 !
 interface eth2
  description tr2-facing
@@ -27,13 +28,13 @@ router bgp 65100
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  bgp log-neighbor-changes
- neighbor eth1 interface remote-as 65000
+ neighbor 2001:db8:1:10::2 remote-as 65000
  neighbor eth2 interface remote-as 65100
  neighbor eth3 interface remote-as 65100
  neighbor eth4 interface remote-as 65100
  !
  address-family ipv6 unicast
-  neighbor eth1 activate
+  neighbor 2001:db8:1:10::2 activate
   neighbor eth2 activate
   neighbor eth2 next-hop-self
   neighbor eth3 activate
diff --git a/deploy/containerlab/node_files/tr2/frr.conf b/deploy/containerlab/node_files/tr2/frr.conf
@@ -8,6 +8,7 @@ interface lo
 !
 interface eth1
  description sjc-worker-facing
+ ipv6 address 2001:db8:1:20::1/64
 !
 interface eth2
  description tr1-facing
@@ -27,13 +28,13 @@ router bgp 65100
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  bgp log-neighbor-changes
- neighbor eth1 interface remote-as 65000
+ neighbor 2001:db8:1:20::2 remote-as 65000
  neighbor eth2 interface remote-as 65100
  neighbor eth3 interface remote-as 65100
  neighbor eth4 interface remote-as 65100
  !
  address-family ipv6 unicast
-  neighbor eth1 activate
+  neighbor 2001:db8:1:20::2 activate
   neighbor eth2 activate
   neighbor eth2 next-hop-self
   neighbor eth3 activate
diff --git a/deploy/containerlab/node_files/tr3/frr.conf b/deploy/containerlab/node_files/tr3/frr.conf
@@ -20,30 +20,32 @@ interface eth3
 !
 interface eth4
  description iad-rr-worker-facing
+ ipv6 address 2001:db8:1:31::1/64
 !
 interface eth5
  description iad-worker-facing
+ ipv6 address 2001:db8:1:30::1/64
 !
 router bgp 65100
  bgp router-id 10.255.255.102
  no bgp default ipv4-unicast
  no bgp ebgp-requires-policy
  bgp log-neighbor-changes
+ neighbor 2001:db8:1:30::2 remote-as 65000
+ neighbor 2001:db8:1:31::2 remote-as 65000
  neighbor eth1 interface remote-as 65100
  neighbor eth2 interface remote-as 65100
  neighbor eth3 interface remote-as 65100
- neighbor eth4 interface remote-as 65000
- neighbor eth5 interface remote-as 65000
  !
  address-family ipv6 unicast
+  neighbor 2001:db8:1:30::2 activate
+  neighbor 2001:db8:1:31::2 activate
   neighbor eth1 activate
   neighbor eth1 next-hop-self
   neighbor eth2 activate
   neighbor eth2 next-hop-self
   neighbor eth3 activate
   neighbor eth3 next-hop-self
-  neighbor eth4 activate
-  neighbor eth5 activate
   network fc00:0:6::1/128
  exit-address-family
 !
diff --git a/deploy/containerlab/resources/overlay/dfw/daemonset/daemonset-patch.yaml b/deploy/containerlab/resources/overlay/dfw/daemonset/daemonset-patch.yaml
@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: overlay
+spec:
+  template:
+    spec:
+      containers:
+        - name: galactic-router
+          env:
+            - name: BGP_LOCAL_ADDRESS
+              value: "fc00:0:2::1"
diff --git a/deploy/containerlab/resources/overlay/dfw/daemonset/kustomization.yaml b/deploy/containerlab/resources/overlay/dfw/daemonset/kustomization.yaml
@@ -3,3 +3,8 @@ namespace: galactic-system
 resources:
   - ../../base
   - namespace.yaml
+patches:
+  - path: daemonset-patch.yaml
+    target:
+      kind: DaemonSet
+      name: overlay
diff --git a/deploy/containerlab/resources/overlay/iad/daemonset/daemonset-patch.yaml b/deploy/containerlab/resources/overlay/iad/daemonset/daemonset-patch.yaml
@@ -16,3 +16,8 @@ spec:
                     operator: In
                     values:
                       - pop
+      containers:
+        - name: galactic-router
+          env:
+            - name: BGP_LOCAL_ADDRESS
+              value: "fc00:0:4::1"
diff --git a/deploy/containerlab/resources/overlay/sjc/daemonset/daemonset-patch.yaml b/deploy/containerlab/resources/overlay/sjc/daemonset/daemonset-patch.yaml
@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: overlay
+spec:
+  template:
+    spec:
+      containers:
+        - name: galactic-router
+          env:
+            - name: BGP_LOCAL_ADDRESS
+              value: "fc00:0:3::1"
diff --git a/deploy/containerlab/resources/overlay/sjc/daemonset/kustomization.yaml b/deploy/containerlab/resources/overlay/sjc/daemonset/kustomization.yaml
@@ -3,3 +3,8 @@ namespace: galactic-system
 resources:
   - ../../base
   - namespace.yaml
+patches:
+  - path: daemonset-patch.yaml
+    target:
+      kind: DaemonSet
+      name: overlay
diff --git a/deploy/containerlab/resources/underlay/dfw/configmap.yaml b/deploy/containerlab/resources/underlay/dfw/configmap.yaml
@@ -39,20 +39,25 @@ data:
     !
     interface eth1
      description tr1-facing
+     ipv6 address 2001:db8:1:10::2/64
     !
 
     ipv6 route 2001:db8:ff01::/48 Null0
     !
+    route-map SET_SRC permit 10
+     set src fc00:0:2::1
+    !
     router bgp 65000
      bgp router-id 10.255.255.2
      no bgp default ipv4-unicast
      no bgp ebgp-requires-policy
      bgp log-neighbor-changes
-     neighbor eth1 interface remote-as 65100
+     neighbor 2001:db8:1:10::1 remote-as 65100
      !
      address-family ipv6 unicast
-      neighbor eth1 activate
-      neighbor eth1 allowas-in 1
+      neighbor 2001:db8:1:10::1 activate
+      neighbor 2001:db8:1:10::1 allowas-in 1
+      neighbor 2001:db8:1:10::1 route-map SET_SRC in
       network 2001:db8:ff01::/48
       network fc00:0:2::/48
      exit-address-family
diff --git a/deploy/containerlab/resources/underlay/iad-rr/configmap.yaml b/deploy/containerlab/resources/underlay/iad-rr/configmap.yaml
@@ -34,23 +34,24 @@ data:
     log syslog informational
 
     interface lo
-     ! /48 provides a reachable address for galactic-router peers to connect to on port 1179.
+     ! /48 provides a reachable address for galactic-router peers to connect to on port 1790.
      ipv6 address fc00:0:8::1/48
     !
     interface eth1
      description tr3-facing
+     ipv6 address 2001:db8:1:31::2/64
     !
 
     router bgp 65000
      bgp router-id 10.255.255.4
      no bgp default ipv4-unicast
      no bgp ebgp-requires-policy
      bgp log-neighbor-changes
-     neighbor eth1 interface remote-as 65100
+     neighbor 2001:db8:1:31::1 remote-as 65100
      !
      address-family ipv6 unicast
-      neighbor eth1 activate
-      neighbor eth1 allowas-in 1
+      neighbor 2001:db8:1:31::1 activate
+      neighbor 2001:db8:1:31::1 allowas-in 1
       network fc00:0:8::/48
      exit-address-family
     !
diff --git a/deploy/containerlab/resources/underlay/iad/configmap.yaml b/deploy/containerlab/resources/underlay/iad/configmap.yaml
@@ -39,20 +39,25 @@ data:
     !
     interface eth1
      description tr3-facing
+     ipv6 address 2001:db8:1:30::2/64
     !
 
     ipv6 route 2001:db8:ff03::/48 Null0
     !
+    route-map SET_SRC permit 10
+     set src fc00:0:4::1
+    !
     router bgp 65000
      bgp router-id 10.255.255.1
      no bgp default ipv4-unicast
      no bgp ebgp-requires-policy
      bgp log-neighbor-changes
-     neighbor eth1 interface remote-as 65100
+     neighbor 2001:db8:1:30::1 remote-as 65100
      !
      address-family ipv6 unicast
-      neighbor eth1 activate
-      neighbor eth1 allowas-in 1
+      neighbor 2001:db8:1:30::1 activate
+      neighbor 2001:db8:1:30::1 allowas-in 1
+      neighbor 2001:db8:1:30::1 route-map SET_SRC in
       network 2001:db8:ff03::/48
       network fc00:0:4::/48
      exit-address-family
diff --git a/deploy/containerlab/resources/underlay/sjc/configmap.yaml b/deploy/containerlab/resources/underlay/sjc/configmap.yaml
@@ -39,20 +39,25 @@ data:
     !
     interface eth1
      description tr2-facing
+     ipv6 address 2001:db8:1:20::2/64
     !
 
     ipv6 route 2001:db8:ff02::/48 Null0
     !
+    route-map SET_SRC permit 10
+     set src fc00:0:3::1
+    !
     router bgp 65000
      bgp router-id 10.255.255.3
      no bgp default ipv4-unicast
      no bgp ebgp-requires-policy
      bgp log-neighbor-changes
-     neighbor eth1 interface remote-as 65100
+     neighbor 2001:db8:1:20::1 remote-as 65100
      !
      address-family ipv6 unicast
-      neighbor eth1 activate
-      neighbor eth1 allowas-in 1
+      neighbor 2001:db8:1:20::1 activate
+      neighbor 2001:db8:1:20::1 allowas-in 1
+      neighbor 2001:db8:1:20::1 route-map SET_SRC in
       network 2001:db8:ff02::/48
       network fc00:0:3::/48
      exit-address-family
diff --git a/internal/runtime/gobgp/peers.go b/internal/runtime/gobgp/peers.go
@@ -57,7 +57,8 @@ func familyFromModel(af model.AddressFamily) *api.Family {
 }
 
 // peerFromDesired converts a DesiredPeer to a GoBGP api.Peer.
-func peerFromDesired(p model.DesiredPeer) *api.Peer {
+// localAddress, if non-empty, is set as the TCP source address for the session.
+func peerFromDesired(p model.DesiredPeer, localAddress string) *api.Peer {
 	peer := &api.Peer{
 		Conf: &api.PeerConf{
 			NeighborAddress: p.Address,
@@ -86,9 +87,12 @@ func peerFromDesired(p model.DesiredPeer) *api.Peer {
 
 	// Connect on the overlay BGP port (1790). Port 179 is occupied by the
 	// underlay FRR bgpd on every node, so GoBGP uses a non-conflicting port.
+	// LocalAddress pins the TCP source to the node's SRv6 loopback so the
+	// return path from the RR uses a routed prefix instead of the link address.
 	peer.Transport = &api.Transport{
-		RemotePort:  1790,
-		PassiveMode: false,
+		RemotePort:   1790,
+		PassiveMode:  false,
+		LocalAddress: localAddress,
 	}
 
 	return peer
diff --git a/internal/runtime/gobgp/runtime.go b/internal/runtime/gobgp/runtime.go
