Skip to content

Commit 7d8efb7

Browse files
cootcoot
committed
Support multiple interfaces
Diffusion now supports multiple interfaces. Outbound connections will use a random interface. On all given addresses / systemd sockets we will run a server accepting connections. `Configuration` type now accepts `dcAddresses :: [Either ntnFd ntnAddr]` which is a list of supported addresses / systemd sockets. One can use `readIPAndPort` parser to parse addresses on a command line.
1 parent b8a53a6 commit 7d8efb7

9 files changed

Lines changed: 111 additions & 86 deletions

File tree

‎ouroboros-network/changelog.d/20260803_173319_coot_diffusion_addresses.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<!--
2+
A new scriv changelog fragment.
3+
4+
Uncomment the section that is right (remove the HTML comment wrapper).
5+
For top level release notes, leave all the headers commented out.
6+
-->
7+
8+
### Breaking
9+
10+
- `Ouroboros.Network.Diffusion.Configuration` now has a single `dcAddresses ::
11+
[Either ntnFd ntnAddr]` field, instead of the two `dcIPv[46]Address`. This
12+
allows us to support multiple interfaces. Use
13+
`Ouroboros.Network.Diffusion.readIPAddressAndPort` to parse `IP:Port` pari
14+
from a command line.
15+
16+
<!--
17+
### Non-Breaking
18+
19+
- A bullet item for the Non-Breaking category.
20+
21+
-->
22+
<!--
23+
### Patch
24+
25+
- A bullet item for the Patch category.
26+
27+
-->

‎ouroboros-network/demo/connection-manager.hs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ withBidirectionalConnectionManager
198198
-> CM.ConnStateIdSupply m
199199
-> DiffTime -- protocol idle timeout
200200
-> DiffTime -- wait time timeout
201-
-> Maybe peerAddr
201+
-> [peerAddr]
202202
-> Random.StdGen
203203
-> ClientAndServerData
204204
-- ^ series of request possible to do with the bidirectional connection
@@ -214,7 +214,7 @@ withBidirectionalConnectionManager snocket makeBearer socket
214214
connStateIdSupply
215215
protocolIdleTimeout
216216
timeWaitTimeout
217-
localAddress
217+
localAddresses
218218
stdGen
219219
ClientAndServerData {
220220
hotInitiatorRequests,
@@ -261,8 +261,8 @@ withBidirectionalConnectionManager snocket makeBearer socket
261261
-- ConnectionManagerTrace
262262
tracer = ("cm",) `contramap` debugTracer,
263263
trTracer = ("cm-state",) `contramap` debugTracer,
264-
ipv4Address = localAddress,
265-
ipv6Address = Nothing,
264+
ipv4Address = localAddresses,
265+
ipv6Address = [],
266266
addressType = \_ -> Just IPv4Address,
267267
snocket = snocket,
268268
makeBearer = makeBearer,
@@ -488,7 +488,7 @@ bidirectionalExperiment
488488
withBidirectionalConnectionManager
489489
snocket makeBearer socket0 connStateIdSupply
490490
protocolIdleTimeout timeWaitTimeout
491-
(Just localAddr) stdGen clientAndServerData $
491+
[localAddr] stdGen clientAndServerData $
492492
\connectionManager _serverAddr _inbGovAsync -> forever' $ do
493493
-- runInitiatorProtocols returns a list of results per each protocol
494494
-- in each bucket (warm \/ hot \/ established); but we run only one

‎ouroboros-network/framework/lib/Ouroboros/Network/ConnectionManager/Core.hs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,14 +95,14 @@ data Arguments handlerTrace socket peerAddr handle handleError versionNumber ver
9595
-- bidirectional @TCP@ connections, it must be the same as the server
9696
-- listening @IPv4@ address.
9797
--
98-
ipv4Address :: Maybe peerAddr,
98+
ipv4Address :: [peerAddr],
9999

100100
-- | @IPv6@ address of the connection manager. If given, outbound
101101
-- connections to an @IPv6@ address will bound to it. To use
102102
-- bidirectional @TCP@ connections, it must be the same as the server
103103
-- listening @IPv6@ address.
104104
--
105-
ipv6Address :: Maybe peerAddr,
105+
ipv6Address :: [peerAddr],
106106

107107
addressType :: peerAddr -> Maybe AddressType,
108108

@@ -1470,10 +1470,10 @@ with args@Arguments {
14701470
)
14711471
$ \socket -> do
14721472
traceWith tracer (TrConnectionNotFound provenance peerAddr)
1473-
let addr = case addressType peerAddr of
1474-
Nothing -> Nothing
1475-
Just IPv4Address -> ipv4Address
1476-
Just IPv6Address -> ipv6Address
1473+
addr <- case addressType peerAddr of
1474+
Nothing -> pure Nothing
1475+
Just IPv4Address -> randomElement stdGenVar ipv4Address
1476+
Just IPv6Address -> randomElement stdGenVar ipv6Address
14771477
configureSocket socket addr
14781478
-- only bind to the ip address if:
14791479
-- the diffusion is given `ipv4/6` addresses;
@@ -2444,3 +2444,13 @@ data Trace peerAddr handlerTrace
24442444
| TrUnexpectedlyFalseAssertion (AssertionLocation peerAddr)
24452445
-- ^ This case is unexpected at call site.
24462446
deriving Show
2447+
2448+
2449+
randomElement :: MonadSTM m
2450+
=> StrictTVar m StdGen -> [a] -> m (Maybe a)
2451+
randomElement _ [] = pure Nothing
2452+
randomElement _ [a] = pure $ Just a
2453+
randomElement stdGenVar as = do
2454+
stdGen <- atomically $ stateTVar stdGenVar Random.splitGen
2455+
let (indx, _) = Random.uniformR (0, length as - 1) stdGen
2456+
return $ Just $ as List.!! indx

‎ouroboros-network/framework/sim-tests/Test/Ouroboros/Network/ConnectionManager.hs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -725,10 +725,10 @@ prop_valid_transitions (Fixed rnd) (SkewedBool bindToLocalAddress) scheduleMap =
725725
in counterexample ("\nTransition Trace\n" ++ (intercalate "\n" . map show $ cmTrace))
726726
(verifyTrace cmTrace)
727727
where
728-
myAddress :: Maybe Addr
728+
myAddress :: [Addr]
729729
myAddress = if bindToLocalAddress
730-
then Just (TestAddress 0)
731-
else Nothing
730+
then [TestAddress 0]
731+
else []
732732

733733
verifyTrace :: [TestAbstractTransitionTrace] -> Property
734734
verifyTrace = conjoin
@@ -771,7 +771,7 @@ prop_valid_transitions (Fixed rnd) (SkewedBool bindToLocalAddress) scheduleMap =
771771
tracer,
772772
trTracer,
773773
ipv4Address = myAddress,
774-
ipv6Address = Nothing,
774+
ipv6Address = [],
775775
addressType = \_ -> Just IPv4Address,
776776
snocket = snocket,
777777
makeBearer = makeFDBearer,

‎ouroboros-network/framework/sim-tests/Test/Ouroboros/Network/Server/Sim.hs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -767,7 +767,7 @@ multinodeExperiment inboundTrTracer trTracer inboundTracer debugTracer cmTracer
767767
( withInitiatorOnlyConnectionManager
768768
name simTimeouts nullTracer cmTracer stdGen
769769
snocket makeBearer connStateIdSupply
770-
(Just localAddr) (mkNextRequests connVar)
770+
[localAddr] (mkNextRequests connVar)
771771
timeLimitsHandshake acceptedConnLimit
772772
( \ connectionManager ->
773773
connectionLoop SingInitiatorMode localAddr cc connectionManager Map.empty connVar
@@ -805,7 +805,7 @@ multinodeExperiment inboundTrTracer trTracer inboundTracer debugTracer cmTracer
805805
inboundTracer muxTracers debugTracer
806806
stdGen
807807
snocket makeBearer connStateIdSupply
808-
(\_ -> pure ()) fd (Just localAddr) serverAcc
808+
(\_ -> pure ()) fd [localAddr] serverAcc
809809
(mkNextRequests connVar)
810810
timeLimitsHandshake
811811
acceptedConnLimit
@@ -823,7 +823,7 @@ multinodeExperiment inboundTrTracer trTracer inboundTracer debugTracer cmTracer
823823
Unidirectional ->
824824
Job ( withInitiatorOnlyConnectionManager
825825
name simTimeouts trTracer cmTracer stdGen snocket makeBearer
826-
connStateIdSupply (Just localAddr)
826+
connStateIdSupply [localAddr]
827827
(mkNextRequests connVar)
828828
timeLimitsHandshake
829829
acceptedConnLimit
@@ -2267,7 +2267,7 @@ prop_server_accept_error (Fixed rnd) (AbsIOError ioerr) =
22672267
makeFDBearer
22682268
connStateIdSupply
22692269
(\_ -> pure ())
2270-
socket0 (Just addr)
2270+
socket0 [addr]
22712271
[accumulatorInit pdata]
22722272
nextRequests
22732273
noTimeLimitsHandshake

‎ouroboros-network/framework/tests-lib/Test/Ouroboros/Network/ConnectionManager/Experiments.hs‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ withInitiatorOnlyConnectionManager
268268
-- ^ series of request possible to do with the bidirectional connection
269269
-- manager towards some peer.
270270
-> CM.ConnStateIdSupply m
271-
-> Maybe peerAddr
271+
-> [peerAddr]
272272
-> TemperatureBundle (ConnectionId peerAddr -> STM m [req])
273273
-- ^ Functions to get the next requests for a given connection
274274
-> ProtocolTimeLimits (Handshake UnversionedProtocol Term)
@@ -280,7 +280,7 @@ withInitiatorOnlyConnectionManager
280280
-> m a)
281281
-> m a
282282
withInitiatorOnlyConnectionManager name timeouts trTracer tracer stdGen snocket makeBearer connStateIdSupply
283-
localAddr nextRequests handshakeTimeLimits acceptedConnLimit k = do
283+
localAddrs nextRequests handshakeTimeLimits acceptedConnLimit k = do
284284
mainThreadId <- myThreadId
285285
let muxTracers :: Mx.TracersWithBearer (ConnectionId peerAddr) m
286286
muxTracers = Mx.Tracers {
@@ -317,8 +317,8 @@ withInitiatorOnlyConnectionManager name timeouts trTracer tracer stdGen snocket
317317
trTracer = (WithName name . fmap CM.abstractState)
318318
`contramap` trTracer,
319319
-- This is actually the low level bearer tracer
320-
ipv4Address = localAddr,
321-
ipv6Address = Nothing,
320+
ipv4Address = localAddrs,
321+
ipv6Address = [],
322322
addressType = \_ -> Just IPv4Address,
323323
snocket,
324324
makeBearer,
@@ -459,7 +459,7 @@ withBidirectionalConnectionManager
459459
-> (socket -> m ()) -- ^ configure socket
460460
-> socket
461461
-- ^ listening socket
462-
-> Maybe peerAddr
462+
-> [peerAddr]
463463
-> acc
464464
-- ^ Initial state for the server
465465
-> TemperatureBundle (ConnectionId peerAddr -> STM m [req])
@@ -482,7 +482,7 @@ withBidirectionalConnectionManager name timeouts
482482
stdGen
483483
snocket makeBearer connStateIdSupply
484484
confSock socket
485-
localAddress
485+
localAddresses
486486
accumulatorInit nextRequests
487487
handshakeTimeLimits
488488
acceptedConnLimit k = do
@@ -516,8 +516,8 @@ withBidirectionalConnectionManager name timeouts
516516
trTracer = (WithName name . fmap CM.abstractState)
517517
`contramap` trTracer,
518518
-- low level bearer tracer
519-
ipv4Address = localAddress,
520-
ipv6Address = Nothing,
519+
ipv4Address = localAddresses,
520+
ipv6Address = [],
521521
addressType = \_ -> Just IPv4Address,
522522
snocket,
523523
makeBearer,
@@ -765,15 +765,15 @@ unidirectionalExperiment stdGen timeouts snocket makeBearer confSock socket clie
765765
nextReqs <- oneshotNextRequests clientAndServerData
766766
connStateIdSupply <- atomically $ CM.newConnStateIdSupply (Proxy @m)
767767
withInitiatorOnlyConnectionManager
768-
"client" timeouts nullTracer nullTracer stdGen' snocket makeBearer connStateIdSupply Nothing nextReqs
768+
"client" timeouts nullTracer nullTracer stdGen' snocket makeBearer connStateIdSupply [] nextReqs
769769
timeLimitsHandshake maxAcceptedConnectionsLimit
770770
$ \connectionManager ->
771771
withBidirectionalConnectionManager "server" timeouts
772772
nullTracer nullTracer nullTracer
773773
nullTracer Mx.nullTracers nullTracer
774774
stdGen''
775775
snocket makeBearer connStateIdSupply
776-
confSock socket Nothing
776+
confSock socket []
777777
[accumulatorInit clientAndServerData]
778778
noNextRequests
779779
timeLimitsHandshake
@@ -859,7 +859,7 @@ bidirectionalExperiment
859859
nullTracer nullTracer nullTracer nullTracer Mx.nullTracers
860860
nullTracer stdGen' snocket makeBearer
861861
connStateIdSupply confSock
862-
socket0 (Just localAddr0)
862+
socket0 [localAddr0]
863863
[accumulatorInit clientAndServerData0]
864864
nextRequests0
865865
noTimeLimitsHandshake
@@ -869,7 +869,7 @@ bidirectionalExperiment
869869
nullTracer nullTracer nullTracer nullTracer Mx.nullTracers
870870
nullTracer stdGen'' snocket makeBearer
871871
connStateIdSupply confSock
872-
socket1 (Just localAddr1)
872+
socket1 [localAddr1]
873873
[accumulatorInit clientAndServerData1]
874874
nextRequests1
875875
noTimeLimitsHandshake

‎ouroboros-network/lib/Ouroboros/Network/Diffusion.hs‎

Lines changed: 28 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ module Ouroboros.Network.Diffusion
1717
, mkInterfaces
1818
, socketAddressType
1919
, module Ouroboros.Network.Diffusion.Types
20-
-- * Utils
20+
-- * Utils
2121
, readIPAndPort
2222
) where
2323

@@ -43,6 +43,7 @@ import Data.Bits ((.|.))
4343
import System.Posix.Files qualified as Unix
4444
#endif
4545
import Data.ByteString.Lazy (ByteString)
46+
import Data.Either (partitionEithers)
4647
import Data.Hashable (Hashable)
4748
import Data.IP qualified as IP
4849
import Data.List.NonEmpty (NonEmpty (..))
@@ -86,9 +87,9 @@ import Ouroboros.Network.PeerSharing (PeerSharingRegistry (..))
8687
import Ouroboros.Network.Protocol.Handshake
8788
import Ouroboros.Network.RethrowPolicy
8889
import Ouroboros.Network.Server qualified as Server
89-
import Ouroboros.Network.Snocket (LocalAddress (..), LocalSocket (..),
90-
RemoteAddress, localSocketFileDescriptor, makeLocalBearer,
91-
makeSocketBearer')
90+
import Ouroboros.Network.Snocket (AddressFamily (..), LocalAddress (..),
91+
LocalSocket (..), RemoteAddress, localSocketFileDescriptor,
92+
makeLocalBearer, makeSocketBearer')
9293
import Ouroboros.Network.Snocket qualified as Snocket
9394
import Ouroboros.Network.Socket (configureSocket, configureSystemdSocket)
9495
import Ouroboros.Network.Util (PrettyShow (..))
@@ -223,8 +224,7 @@ runM Interfaces
223224
, daSRVPrefix
224225
}
225226
Configuration
226-
{ dcIPv4Address
227-
, dcIPv6Address
227+
{ dcAddresses
228228
, dcLocalAddress
229229
, dcAcceptedConnectionsLimit
230230
, dcMode = diffusionMode
@@ -371,8 +371,8 @@ runM Interfaces
371371
CM.with CM.Arguments {
372372
CM.tracer = dtLocalConnectionManagerTracer,
373373
CM.trTracer = nullTracer, -- TODO: issue #3320
374-
CM.ipv4Address = Nothing,
375-
CM.ipv6Address = Nothing,
374+
CM.ipv4Address = [],
375+
CM.ipv6Address = [],
376376
CM.addressType = const Nothing,
377377
CM.snocket = diNtcSnocket,
378378
CM.makeBearer = diNtcBearer,
@@ -426,31 +426,23 @@ runM Interfaces
426426
epErrorDelay = daRepromoteErrorDelay
427427
}
428428

429-
ipv4Address
430-
<- traverse (either (Snocket.getLocalAddr diNtnSnocket) pure)
431-
dcIPv4Address
432-
case ipv4Address of
433-
Just addr | Just IPv4Address <- diNtnAddressType addr
434-
-> pure ()
435-
| otherwise
436-
-> throwIO (UnexpectedIPv4Address addr)
437-
Nothing -> pure ()
438-
439-
ipv6Address
440-
<- traverse (either (Snocket.getLocalAddr diNtnSnocket) pure)
441-
dcIPv6Address
442-
case ipv6Address of
443-
Just addr | Just IPv6Address <- diNtnAddressType addr
444-
-> pure ()
445-
| otherwise
446-
-> throwIO (UnexpectedIPv6Address addr)
447-
Nothing -> pure ()
448-
449-
lookupReqs <- case (ipv4Address, ipv6Address) of
450-
(Just _ , Nothing) -> return RootPeersDNS.LookupReqAOnly
451-
(Nothing, Just _ ) -> return RootPeersDNS.LookupReqAAAAOnly
452-
(Just _ , Just _ ) -> return RootPeersDNS.LookupReqAAndAAAA
453-
(Nothing, Nothing) -> throwIO NoSocket
429+
addrs <- traverse (either (Snocket.getLocalAddr diNtnSnocket) pure) dcAddresses
430+
let (ipv4Addresses, ipv6Addresses) =
431+
partitionEithers
432+
$ catMaybes
433+
$ map (\addr ->
434+
case Snocket.addrFamily diNtnSnocket addr of
435+
SocketFamily Socket.AF_INET -> Just (Left addr)
436+
SocketFamily Socket.AF_INET6 -> Just (Right addr)
437+
_ -> Nothing
438+
)
439+
$ addrs
440+
441+
lookupReqs <- case (ipv4Addresses, ipv6Addresses) of
442+
(_:_, [] ) -> return RootPeersDNS.LookupReqAOnly
443+
([] , _:_) -> return RootPeersDNS.LookupReqAAAAOnly
444+
(_:_, _:_) -> return RootPeersDNS.LookupReqAAndAAAA
445+
([] , [] ) -> throwIO NoSocket
454446

455447
localRootsVar <- newTVarIO mempty
456448

@@ -495,8 +487,8 @@ runM Interfaces
495487
CM.trTracer =
496488
fmap CM.abstractState
497489
`contramap` dtConnectionManagerTransitionTracer,
498-
CM.ipv4Address,
499-
CM.ipv6Address,
490+
CM.ipv4Address = ipv4Addresses,
491+
CM.ipv6Address = ipv6Addresses,
500492
CM.addressType = diNtnAddressType,
501493
CM.snocket = diNtnSnocket,
502494
CM.makeBearer = diNtnBearer,
@@ -729,11 +721,7 @@ runM Interfaces
729721
withSockets tracer diNtnSnocket
730722
(\sock addr -> diNtnConfigureSocket sock (Just addr))
731723
(\sock addr -> diNtnConfigureSystemdSocket sock addr)
732-
( catMaybes
733-
[ dcIPv4Address
734-
, dcIPv6Address
735-
]
736-
)
724+
dcAddresses
737725
f
738726

739727
-- run node-to-node server

0 commit comments

Comments
 (0)