diff --git a/Applications/Quickstarts.Servers/DurableSubscription/SubscriptionStore.cs b/Applications/Quickstarts.Servers/DurableSubscription/SubscriptionStore.cs index 3640f57588..47efaf94a7 100644 --- a/Applications/Quickstarts.Servers/DurableSubscription/SubscriptionStore.cs +++ b/Applications/Quickstarts.Servers/DurableSubscription/SubscriptionStore.cs @@ -31,6 +31,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Opc.Ua; using Opc.Ua.Server; @@ -56,7 +58,14 @@ public SubscriptionStore(IServerInternal server) .MonitoredItemQueueFactory as DurableMonitoredItemQueueFactory; } - public bool StoreSubscriptions(IEnumerable subscriptions) + public ValueTask StoreSubscriptionsAsync( + IEnumerable subscriptions, + CancellationToken cancellationToken = default) + { + return new ValueTask(StoreSubscriptionsCore(subscriptions)); + } + + private bool StoreSubscriptionsCore(IEnumerable subscriptions) { try { @@ -99,7 +108,13 @@ public bool StoreSubscriptions(IEnumerable subscriptions) return false; } - public RestoreSubscriptionResult RestoreSubscriptions() + public ValueTask RestoreSubscriptionsAsync( + CancellationToken cancellationToken = default) + { + return new ValueTask(RestoreSubscriptionsCore()); + } + + private RestoreSubscriptionResult RestoreSubscriptionsCore() { string filePath = Path.Combine(s_storage_path, kFilename); try @@ -152,7 +167,9 @@ public IEventMonitoredItemQueue RestoreEventMonitoredItemQueue(uint monitoredIte s_storage_path)!; } - public void OnSubscriptionRestoreComplete(Dictionary> createdSubscriptions) + public ValueTask OnSubscriptionRestoreCompleteAsync( + Dictionary> createdSubscriptions, + CancellationToken cancellationToken = default) { string filePath = Path.Combine(s_storage_path, kFilename); @@ -174,6 +191,7 @@ public void OnSubscriptionRestoreComplete(Dictionary> create IEnumerable ids = createdSubscriptions.SelectMany(s => s.Value.Memory.ToArray()); m_durableMonitoredItemQueueFactory.CleanStoredQueues(s_storage_path, ids); } + return default; } public static void EncodeSubscription( diff --git a/Applications/RedundantClient/Dockerfile b/Applications/RedundantClient/Dockerfile new file mode 100644 index 0000000000..f64b9d445d --- /dev/null +++ b/Applications/RedundantClient/Dockerfile @@ -0,0 +1,18 @@ +# OPC UA RedundantClient (managed client) sample image. +# +# The build context must be the repository ROOT so the project references resolve. +# Build directly: +# docker build -f Applications/RedundantClient/Dockerfile -t opcua-redundant-client . +# Or via docker compose (see Applications/RedundantServer/docker-compose.*.yml). +FROM mcr.microsoft.com/dotnet/sdk AS build +WORKDIR /src +ENV DOTNET_EnableWriteXorExecute=0 +COPY . . +RUN dotnet publish "Applications/RedundantClient/RedundantClient.csproj" \ + -c Release -f net10.0 -p:PublishAot=false \ + -o /app/publish + +FROM mcr.microsoft.com/dotnet/runtime AS final +WORKDIR /app +COPY --from=build /app/publish . +ENTRYPOINT ["dotnet", "RedundantClient.dll"] diff --git a/Applications/RedundantClient/Program.cs b/Applications/RedundantClient/Program.cs new file mode 100644 index 0000000000..8c635c50a4 --- /dev/null +++ b/Applications/RedundantClient/Program.cs @@ -0,0 +1,351 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua; +using Opc.Ua.Client; +using Opc.Ua.Client.Redundancy; +using Opc.Ua.Configuration; +using Opc.Ua.Redundancy; + +namespace RedundantClient +{ + /// + /// Entry point for the managed client sample. + /// + public static class Program + { + /// + /// Starts the sample. + /// + public static Task Main(string[] args) + { + var serverOption = new Option("--server", "-s") + { + Description = "Discovery URL of any server in the (optionally) redundant set.", + DefaultValueFactory = _ => "opc.tcp://localhost:62543/RedundantServer" + }; + var noSecurityOption = new Option("--nosecurity") + { + Description = "Select endpoints with MessageSecurityMode.None." + }; + var autoAcceptOption = new Option("--autoaccept") + { + Description = "Automatically accept untrusted server certificates for sample runs." + }; + var durationOption = new Option("--duration", "-d") + { + Description = "How long to monitor before exiting. Use 00:00:00 to run until Ctrl+C.", + DefaultValueFactory = _ => TimeSpan.FromMinutes(2) + }; + var replicasOption = new Option("--replicas") + { + Description = "Run an in-process client replica set of this size (leader holds the session).", + DefaultValueFactory = _ => 1 + }; + + var rootCommand = new RootCommand( + "OPC UA managed client sample that transparently handles server redundancy") + { + serverOption, + noSecurityOption, + autoAcceptOption, + durationOption, + replicasOption + }; + + rootCommand.SetAction(async (parseResult, cancellationToken) => + { + await RunAsync( + parseResult.GetValue(serverOption)!, + parseResult.GetValue(noSecurityOption), + parseResult.GetValue(autoAcceptOption), + parseResult.GetValue(durationOption), + parseResult.GetValue(replicasOption), + cancellationToken).ConfigureAwait(false); + }); + + ParseResult parseResult = rootCommand.Parse(args); + return parseResult.InvokeAsync(new InvocationConfiguration(), CancellationToken.None); + } + + private static async Task RunAsync( + string serverUrl, + bool noSecurity, + bool autoAccept, + TimeSpan duration, + int replicas, + CancellationToken ct) + { + ITelemetryContext telemetry = DefaultTelemetry.Create(builder => + { + builder.SetMinimumLevel(LogLevel.Information); + }); + using IDisposable? telemetryDisposable = telemetry as IDisposable; + + var application = new ApplicationInstance(telemetry) + { + ApplicationName = kApplicationName, + ApplicationType = ApplicationType.Client, + ConfigSectionName = kConfigSectionName, + CertificatePasswordProvider = new CertificatePasswordProvider([]) + }; + + await using (application.ConfigureAwait(false)) + { + ApplicationConfiguration configuration = await application + .LoadApplicationConfigurationAsync(silent: false, ct: ct) + .ConfigureAwait(false); + if (autoAccept) + { + configuration.CertificateManager.AcceptError = (_, _) => true; + } + + bool haveCertificate = await application + .CheckApplicationInstanceCertificatesAsync(silent: false, ct: ct) + .ConfigureAwait(false); + if (!haveCertificate) + { + throw new InvalidOperationException("Application instance certificate invalid."); + } + + EndpointDescription selectedEndpoint = await CoreClientUtils + .SelectEndpointAsync(configuration, serverUrl, useSecurity: !noSecurity, telemetry, ct) + .ConfigureAwait(false) + ?? throw new InvalidOperationException( + $"No endpoint could be selected for '{serverUrl}'."); + var endpoint = new ConfiguredEndpoint( + null, + selectedEndpoint, + EndpointConfiguration.Create(configuration)); + + Console.WriteLine("Connecting managed client to {0}", serverUrl); + + if (replicas > 1) + { + await RunReplicaSetAsync( + configuration, endpoint, telemetry, replicas, duration, ct).ConfigureAwait(false); + return; + } + + // A single ManagedSession is the managed client. WithServerRedundancy() lets it + // discover the redundant set (if any) from the connected server and fail over + // transparently; against a server that is not configured for redundancy it simply + // behaves as a resilient reconnecting session. The caller does not need to know the + // server topology before connecting. + ManagedSession session = await new ManagedSessionBuilder(configuration, telemetry) + .UseEndpoint(endpoint) + .WithSessionName(kApplicationName) + .WithUserIdentity(new UserIdentity()) + .WithServerRedundancy() + .ConnectAsync(ct) + .ConfigureAwait(false); + + await using (session.ConfigureAwait(false)) + { + session.ConnectionStateChanged += OnConnectionStateChanged; + + await LogRedundancyInfoAsync(session, ct).ConfigureAwait(false); + await SubscribeToCurrentTimeAsync(session, ct).ConfigureAwait(false); + + Console.WriteLine("Monitoring ServerStatus.CurrentTime. Press Ctrl+C to stop."); + await RunForDurationAsync(duration, ct).ConfigureAwait(false); + + session.ConnectionStateChanged -= OnConnectionStateChanged; + } + } + } + + private static async Task LogRedundancyInfoAsync(ManagedSession session, CancellationToken ct) + { + var handler = new DefaultServerRedundancyHandler(); + ServerRedundancyInfo info = await handler + .FetchRedundancyInfoAsync(session, ct) + .ConfigureAwait(false); + if (info.Mode == RedundancySupport.None) + { + Console.WriteLine( + "Server is not configured for redundancy (RedundancySupport=None); " + + "running as a single resilient session."); + return; + } + + Console.WriteLine( + "Server reports RedundancySupport={0}, ServiceLevel={1} ({2}), CurrentServerId={3}.", + info.Mode, + info.ServiceLevel, + info.ServiceLevelSubrange, + info.CurrentServerId); + for (int ii = 0; ii < info.RedundantServers.Count; ii++) + { + RedundantServer server = info.RedundantServers[ii]; + Console.WriteLine( + "Peer {0}: uri={1}, state={2}, serviceLevel={3}, endpoint={4}", + ii + 1, + server.ServerUri, + server.ServerState, + server.ServiceLevel, + server.Endpoint?.EndpointUrl?.ToString() ?? "(unresolved)"); + } + } + + private static async Task SubscribeToCurrentTimeAsync(ManagedSession session, CancellationToken ct) + { + // Ownership of the subscription transfers to the session via AddSubscription; + // the session disposes its subscriptions when it is disposed. +#pragma warning disable CA2000 + var subscription = new Subscription(session.DefaultSubscription) + { + DisplayName = "RedundantClient CurrentTime", + PublishingEnabled = true, + PublishingInterval = 1000, + KeepAliveCount = 10, + LifetimeCount = 0, + MinLifetimeInterval = 10_000, + FastDataChangeCallback = OnDataChange + }; + session.AddSubscription(subscription); +#pragma warning restore CA2000 + await subscription.CreateAsync(ct).ConfigureAwait(false); + + var currentTime = new MonitoredItem(subscription.DefaultItem) + { + StartNodeId = VariableIds.Server_ServerStatus_CurrentTime, + AttributeId = Attributes.Value, + DisplayName = "ServerStatus.CurrentTime", + SamplingInterval = 1000, + QueueSize = 10, + DiscardOldest = true + }; + subscription.AddItem(currentTime); + await subscription.ApplyChangesAsync(ct).ConfigureAwait(false); + } + + private static void OnDataChange( + Subscription subscription, + DataChangeNotification notification, + ArrayOf stringTable) + { + for (int ii = 0; ii < notification.MonitoredItems.Count; ii++) + { + MonitoredItemNotification item = notification.MonitoredItems[ii]; + Console.WriteLine( + "CurrentTime={0:o} Status={1}", + item.Value.GetValue(DateTime.MinValue), + item.Value.StatusCode); + } + } + + private static void OnConnectionStateChanged(object? sender, ConnectionStateChangedEventArgs e) + { + Console.WriteLine("Connection state: {0} -> {1}", e.PreviousState, e.NewState); + } + + private static async Task RunForDurationAsync(TimeSpan duration, CancellationToken ct) + { + try + { + await Task.Delay( + duration <= TimeSpan.Zero ? Timeout.InfiniteTimeSpan : duration, + ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Ctrl+C or the run duration elapsed; exit cleanly. + } + } + + private static async Task RunReplicaSetAsync( + ApplicationConfiguration configuration, + ConfiguredEndpoint endpoint, + ITelemetryContext telemetry, + int replicas, + TimeSpan duration, + CancellationToken ct) + { + // A shared store + lease election make exactly one replica the leader that holds the + // session; followers stand by and take over on leader loss. This runs in-process with an + // in-memory store; a multi-process deployment uses a CAS-capable shared store (Redis) or + // Kubernetes Lease election with the same coordinator. + using var store = new InMemorySharedKeyValueStore(); + var coordinators = new List(); + try + { + for (int i = 0; i < replicas; i++) + { + string nodeId = $"replica-{i + 1}"; + // Ownership of the election transfers to the coordinator, which disposes it. +#pragma warning disable CA2000 + var election = new SharedStoreLeaseElection( + store, "client-replica/leader", nodeId, + TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(5), TimeProvider.System); +#pragma warning restore CA2000 + var options = new ClientReplicaOptions + { + NodeId = nodeId, + Mode = ClientStandbyMode.Cold, + CreateSessionAsync = token => new ValueTask( + new ManagedSessionBuilder(configuration, telemetry) + .UseEndpoint(endpoint) + .WithSessionName(nodeId) + .WithUserIdentity(new UserIdentity()) + .ConnectAsync(token)) + }; + var coordinator = new ClientReplicaCoordinator( + options, election, store, NullRecordProtector.Instance, telemetry); + coordinator.RoleChanged += isLeader => + Console.WriteLine("{0} is now {1}", nodeId, isLeader ? "LEADER" : "follower"); + coordinators.Add(coordinator); + await coordinator.StartAsync(ct).ConfigureAwait(false); + } + + Console.WriteLine("Client replica set of {0} started; the leader holds the session.", replicas); + await RunForDurationAsync(duration, ct).ConfigureAwait(false); + } + finally + { + foreach (ClientReplicaCoordinator coordinator in coordinators) + { + await coordinator.DisposeAsync().ConfigureAwait(false); + } + } + } + + private const string kApplicationName = "RedundantClient"; + private const string kConfigSectionName = "RedundantClient"; + } +} diff --git a/Applications/RedundantClient/Properties/AssemblyInfo.cs b/Applications/RedundantClient/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..41cfff37eb --- /dev/null +++ b/Applications/RedundantClient/Properties/AssemblyInfo.cs @@ -0,0 +1,34 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +#nullable enable + +[assembly: CLSCompliant(false)] diff --git a/Applications/RedundantClient/README.md b/Applications/RedundantClient/README.md new file mode 100644 index 0000000000..97f3af9433 --- /dev/null +++ b/Applications/RedundantClient/README.md @@ -0,0 +1,34 @@ +# Managed client sample + +This console sample shows the recommended way to connect an OPC UA client: build a single `ManagedSession` with `WithServerRedundancy()`. The same code works whether or not the target server is configured for redundancy, because a client is almost never aware of the server topology until it has connected. + +- Against a redundant server, the managed session reads `Server.ServerRedundancy` / `Server.ServiceLevel`, discovers the redundant set from the connected server, and fails over transparently. There is no client-side failover-mode selection and no hand-maintained seed list — the peer set comes from the server. +- Against a server that is not configured for redundancy, the same session simply runs as a resilient, automatically reconnecting client (`RedundancySupport=None`). + +## Run + +```powershell +dotnet run --project Applications\RedundantClient\RedundantClient.csproj -- ` + --server opc.tcp://localhost:62543/RedundantServer --autoaccept --nosecurity --duration 00:05:00 +``` + +| Option | Default | Description | +| --- | --- | --- | +| `--server`, `-s` | `opc.tcp://localhost:62543/RedundantServer` | Discovery URL of any server in the (optionally) redundant set. | +| `--nosecurity` | off | Select endpoints with `MessageSecurityMode.None`. | +| `--autoaccept` | off | Automatically accept untrusted server certificates (sample only). | +| `--duration`, `-d` | `00:02:00` | How long to monitor before exiting; `00:00:00` runs until Ctrl+C. | + +The sample connects, logs the server's reported `RedundancySupport` (or notes that the server is not redundant), subscribes to `Server.ServerStatus.CurrentTime`, and logs the values together with any transparent connection-state changes (reconnect or failover). To observe failover, lower the active server's service level (for example with the `RedundantServer` sample's manual failover support) or stop the active server; the managed session reconnects to a healthy peer on its own. + +See [HighAvailability.md](../../Docs/HighAvailability.md) for the redundancy design and the [RedundantServer](../RedundantServer/README.md) sample for the server side. + +## Client replica set (high availability) + +Run an in-process client replica set where exactly one leader holds the session and followers stand by: + +```powershell +dotnet run --project Applications\RedundantClient\RedundantClient.csproj -- --server opc.tcp://localhost:62543/RedundantServer --autoaccept --nosecurity --replicas 3 +``` + +Leader election uses a shared `ISharedKeyValueStore` + `SharedStoreLeaseElection` (in-memory here). A multi-process deployment uses a CAS-capable shared store (Redis) or Kubernetes Lease election with the same `ClientReplicaCoordinator`. diff --git a/Applications/RedundantClient/RedundantClient.Config.xml b/Applications/RedundantClient/RedundantClient.Config.xml new file mode 100644 index 0000000000..45b87773b6 --- /dev/null +++ b/Applications/RedundantClient/RedundantClient.Config.xml @@ -0,0 +1,69 @@ + + + Redundant Client Sample + urn:localhost:UA:RedundantClient + uri:opcfoundation.org:RedundantClient + Client_1 + + + + Directory + %LocalApplicationData%/OPC Foundation/pki/own + CN=Redundant Client Sample, C=US, S=Arizona, O=OPC Foundation, DC=localhost + RsaSha256 + + + + Directory + %LocalApplicationData%/OPC Foundation/pki/issuer + + + Directory + %LocalApplicationData%/OPC Foundation/pki/trusted + + + Directory + %LocalApplicationData%/OPC Foundation/pki/rejected + + false + true + true + 2048 + false + true + + + + 120000 + 4194304 + 4194304 + 65535 + 4194304 + 65535 + 300000 + 3600000 + + + 60000 + + opc.tcp://{0}:4840 + + + 10000 + + 2500 + 2500 + 2500 + 2500 + + + + + %LocalApplicationData%/OPC Foundation/Logs/RedundantClient.log.txt + true + + diff --git a/Applications/RedundantClient/RedundantClient.csproj b/Applications/RedundantClient/RedundantClient.csproj new file mode 100644 index 0000000000..500e5fe506 --- /dev/null +++ b/Applications/RedundantClient/RedundantClient.csproj @@ -0,0 +1,30 @@ + + + net10.0 + Exe + RedundantClient + RedundantClient + OPC Foundation + OPC UA non-transparent redundant managed client sample. + Copyright © 2004-2025 OPC Foundation, Inc + RedundantClient + enable + false + true + true + + + + + + + + + + + + + Always + + + diff --git a/Applications/RedundantServer/Dockerfile b/Applications/RedundantServer/Dockerfile new file mode 100644 index 0000000000..4c53b05f08 --- /dev/null +++ b/Applications/RedundantServer/Dockerfile @@ -0,0 +1,20 @@ +# OPC UA RedundantServer sample image. +# +# The build context must be the repository ROOT so the project references resolve. +# Build directly: +# docker build -f Applications/RedundantServer/Dockerfile -t opcua-redundant-server . +# Or via docker compose (see Applications/RedundantServer/docker-compose.*.yml). +FROM mcr.microsoft.com/dotnet/sdk AS build +WORKDIR /src +ENV DOTNET_EnableWriteXorExecute=0 +COPY . . +RUN dotnet publish "Applications/RedundantServer/RedundantServer.csproj" \ + -c Release -f net10.0 -p:PublishAot=false -p:CustomTestTarget=net10.0 \ + -o /app/publish + +FROM mcr.microsoft.com/dotnet/runtime AS final +WORKDIR /app +COPY --from=build /app/publish . +# OPC UA endpoint port and (active/active) address-space + session gossip ports. +EXPOSE 62543 4840 4841 +ENTRYPOINT ["dotnet", "RedundantServer.dll"] diff --git a/Applications/RedundantServer/HaSampleNodeManager.cs b/Applications/RedundantServer/HaSampleNodeManager.cs new file mode 100644 index 0000000000..e2a4c3d986 --- /dev/null +++ b/Applications/RedundantServer/HaSampleNodeManager.cs @@ -0,0 +1,275 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; +using Opc.Ua.Redundancy.Server; + +namespace RedundantServer +{ + /// + /// Factory that produces instances for DI-hosted servers. + /// + public sealed class HaSampleNodeManagerFactory : IAsyncNodeManagerFactory + { + private const string NamespaceUri = "http://opcfoundation.org/UA/Samples/HighAvailability"; + private readonly ILeaderElection m_leaderElection; + private readonly HaSampleReplicaInfo m_replicaInfo; + + /// + /// Creates a factory using the distributed leader-election service registered by the host. + /// + /// The leader-election service that identifies the active writer replica. + /// The local replica identity published by sample variables. + public HaSampleNodeManagerFactory(ILeaderElection leaderElection, HaSampleReplicaInfo replicaInfo) + { + m_leaderElection = leaderElection ?? throw new ArgumentNullException(nameof(leaderElection)); + m_replicaInfo = replicaInfo ?? throw new ArgumentNullException(nameof(replicaInfo)); + } + + /// + public ArrayOf NamespacesUris => [NamespaceUri]; + + /// + public ValueTask CreateAsync( + IServerInternal server, + ApplicationConfiguration configuration, + CancellationToken cancellationToken = default) + { + _ = configuration; + _ = cancellationToken; + +#pragma warning disable CA2000 // ownership transfers to the server + var manager = new HaSampleNodeManager(server, m_leaderElection, m_replicaInfo, [.. NamespacesUris]); +#pragma warning restore CA2000 + return new ValueTask(manager); + } + } + + /// + /// Carries the local replica identity into sample node managers created by dependency injection. + /// + public sealed class HaSampleReplicaInfo + { + /// + /// Creates a replica identity descriptor. + /// + /// The unique local high-availability node id. + public HaSampleReplicaInfo(string nodeId) + { + NodeId = nodeId; + } + + /// + /// Gets the unique local high-availability node id. + /// + public string NodeId { get; } + } + + /// + /// Minimal address space that participates in distributed replication. + /// + public sealed class HaSampleNodeManager : AsyncCustomNodeManager + { + private readonly ILeaderElection m_leaderElection; + private readonly HaSampleReplicaInfo m_replicaInfo; + private readonly CancellationTokenSource m_simulationCts = new(); + private readonly Lock m_updateLock = new(); + private BaseDataVariableState? m_counter; + private BaseDataVariableState? m_activeReplica; + private Task? m_simulationTask; + private int m_counterValue; + + /// + /// Creates the high-availability sample node manager. + /// + /// The server that owns the node manager. + /// The leader-election service that gates sample writes. + /// The local replica identity published by sample variables. + /// The namespace URIs exposed by this node manager. + public HaSampleNodeManager( + IServerInternal server, + ILeaderElection leaderElection, + HaSampleReplicaInfo replicaInfo, + params string[] namespaceUris) + : base(server, namespaceUris) + { + m_leaderElection = leaderElection ?? throw new ArgumentNullException(nameof(leaderElection)); + m_replicaInfo = replicaInfo ?? throw new ArgumentNullException(nameof(replicaInfo)); + } + + /// + public override async ValueTask CreateAddressSpaceAsync( + IDictionary> externalReferences, + CancellationToken cancellationToken = default) + { + if (!externalReferences.TryGetValue(ObjectIds.ObjectsFolder, out IList? references)) + { + externalReferences[ObjectIds.ObjectsFolder] = references = []; + } + + ushort namespaceIndex = NamespaceIndexes[0]; + FolderState folder = CreateFolder(null, namespaceIndex, "HighAvailability", "High Availability"); + folder.AddReference(ReferenceTypeIds.Organizes, true, ObjectIds.ObjectsFolder); + references.Add(new NodeStateReference(ReferenceTypeIds.Organizes, false, folder.NodeId)); + + m_counter = CreateVariable(folder, namespaceIndex, "Counter", DataTypeIds.Int32, Variant.From(0)); + m_activeReplica = CreateVariable( + folder, + namespaceIndex, + "ActiveReplica", + DataTypeIds.String, + Variant.From("unknown")); + + await AddPredefinedNodeAsync(SystemContext, folder, cancellationToken).ConfigureAwait(false); + StartSimulation(); + } + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + m_simulationCts.Cancel(); + m_simulationCts.Dispose(); + } + base.Dispose(disposing); + } + + private static FolderState CreateFolder(NodeState? parent, ushort namespaceIndex, string path, string name) + { + var folder = new FolderState(parent) + { + SymbolicName = name, + ReferenceTypeId = ReferenceTypeIds.Organizes, + TypeDefinitionId = ObjectTypeIds.FolderType, + NodeId = new NodeId(path, namespaceIndex), + BrowseName = new QualifiedName(path, namespaceIndex), + DisplayName = new LocalizedText("en", name), + WriteMask = AttributeWriteMask.None, + UserWriteMask = AttributeWriteMask.None, + EventNotifier = EventNotifiers.None + }; + + parent?.AddChild(folder); + return folder; + } + + private static BaseDataVariableState CreateVariable( + NodeState parent, + ushort namespaceIndex, + string name, + NodeId dataType, + Variant initialValue) + { + var variable = new BaseDataVariableState(parent) + { + SymbolicName = name, + ReferenceTypeId = ReferenceTypeIds.Organizes, + TypeDefinitionId = VariableTypeIds.BaseDataVariableType, + NodeId = new NodeId(name, namespaceIndex), + BrowseName = new QualifiedName(name, namespaceIndex), + DisplayName = new LocalizedText("en", name), + WriteMask = AttributeWriteMask.None, + UserWriteMask = AttributeWriteMask.None, + DataType = dataType, + ValueRank = ValueRanks.Scalar, + AccessLevel = AccessLevels.CurrentReadOrWrite, + UserAccessLevel = AccessLevels.CurrentReadOrWrite, + Historizing = false, + Value = initialValue, + StatusCode = StatusCodes.Good, + Timestamp = DateTime.UtcNow + }; + + parent.AddChild(variable); + return variable; + } + + private void StartSimulation() + { + m_simulationTask ??= Task.Run(() => RunSimulationAsync(m_simulationCts.Token)); + } + + private async Task RunSimulationAsync(CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1)); + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) + { + if (!m_leaderElection.IsLeader) + { + UpdateActiveReplica("standby"); + continue; + } + + UpdateCounter(); + UpdateActiveReplica(m_replicaInfo.NodeId); + } + } + + private void UpdateCounter() + { + BaseDataVariableState? counter = m_counter; + if (counter == null) + { + return; + } + + int value = Interlocked.Increment(ref m_counterValue); + lock (m_updateLock) + { + counter.Value = value; + counter.Timestamp = DateTime.UtcNow; + counter.ClearChangeMasks(SystemContext, false); + } + } + + private void UpdateActiveReplica(string value) + { + BaseDataVariableState? activeReplica = m_activeReplica; + if (activeReplica == null) + { + return; + } + + lock (m_updateLock) + { + activeReplica.Value = value; + activeReplica.Timestamp = DateTime.UtcNow; + activeReplica.ClearChangeMasks(SystemContext, false); + } + } + + } +} \ No newline at end of file diff --git a/Applications/RedundantServer/Program.cs b/Applications/RedundantServer/Program.cs new file mode 100644 index 0000000000..306f913cf5 --- /dev/null +++ b/Applications/RedundantServer/Program.cs @@ -0,0 +1,433 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Net; +using Crdt; +using RedundantServer; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Opc.Ua; +using Opc.Ua.Server; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; +using Raft; +using Raft.Configuration; +using Raft.Storage; +using Raft.Transport.NanoMsg; + +HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); + +builder.Logging.ClearProviders(); +builder.Logging.AddConsole(); + +int port = int.TryParse(builder.Configuration["port"], out int p) ? p : 62543; +string nodeId = builder.Configuration["HA_NODE_ID"] ?? Guid.NewGuid().ToString("N"); +// Host advertised in the endpoint URL. Defaults to localhost; set HA_HOST to the +// reachable hostname (for example the container/service name) for containerized or +// multi-host deployments so peers and clients can connect across the network. +string host = builder.Configuration["HA_HOST"] ?? "localhost"; +string endpointUrl = $"opc.tcp://{host}:{port}/RedundantServer"; +// Transparent redundancy presents ONE logical server: every replica must share +// the same ApplicationUri (CreateSession validates serverUri against it) and the +// same ApplicationInstanceCertificate. Override the per-node identity with the +// shared values when running behind a single virtual endpoint. +string applicationUri = builder.Configuration["HA_APPLICATION_URI"] + ?? $"urn:localhost:OPCFoundation:RedundantServer:{nodeId}"; +// Optional shared certificate: point every replica's PKI store at the same +// (mounted) directory and give the certificate a stable subject so all replicas +// load one ApplicationInstanceCertificate. In production this is provisioned from +// a Kubernetes Secret / KMS rather than a shared volume. +string? sharedSubjectName = builder.Configuration["HA_SUBJECT_NAME"]; +string? sharedPkiRoot = builder.Configuration["HA_PKI_ROOT"]; + +// Select the redundancy topology: "ap" (active/passive, default — a single +// elected writer) or "aa" (active/active — every replica writes and converges +// by CRDT gossip). +string haMode = (builder.Configuration["HA_MODE"] ?? "ap").Trim().ToLowerInvariant(); +bool activeActive = haMode is "aa" or "activeactive" or "active-active"; +RedundancySupport redundancyMode = ParseRedundancyMode( + builder.Configuration["REDUNDANCY_MODE"], + defaultMode: activeActive ? RedundancySupport.HotAndMirrored : RedundancySupport.Hot); +ArrayOf redundantPeers = ReadRedundantPeers(builder.Configuration); + +builder.Services.AddSingleton(new HaSampleReplicaInfo(nodeId)); + +// Optional: a base64 32-byte master key shared by all replicas (provisioned +// from a Kubernetes Secret / KMS in production). When present, every record +// written to the shared store is encrypted + integrity-protected at rest. +string? recordKeyBase64 = builder.Configuration["HA_RECORD_KEY"]; +byte[]? recordKey = string.IsNullOrWhiteSpace(recordKeyBase64) + ? null + : Convert.FromBase64String(recordKeyBase64); + +// Opt into mirrored fast reconnect (default is the safe re-auth-on-failover). +bool enableFastReconnect = + bool.TryParse(builder.Configuration["HA_FAST_RECONNECT"], out bool fr) && fr; + +// Strong-consistency (Raft) shared store. When HA_CONSISTENCY=strong, a multi-node +// RaftCs cluster (over NanoMsg) backs the shared store, giving real cross-container +// active/passive HA - linearizable leader election and single-use session nonce - +// unlike the default in-memory store, which is private to each container. +string consistency = (builder.Configuration["HA_CONSISTENCY"] ?? "eventual").Trim().ToLowerInvariant(); +bool useStrongConsistency = consistency is "strong"; + +// Optional GetEndpoints load direction: when HA_BALANCING_URL is set, a GetEndpoints +// request on that (virtual/load-balancer) discovery URL is answered with the best +// peer's endpoints - the active server (active/passive) or the least-loaded healthy +// peer (active/active). It complements the standard client-driven ServiceLevel +// selection; plain discovery on this node's own URL is unaffected. +string? balancingUrl = builder.Configuration["HA_BALANCING_URL"]; + +if (activeActive && recordKey != null) +{ + // Encrypt mirrored session entries at rest (sessions are gossiped as a CRDT). + builder.Services.AddSingleton(_ => new AesCbcHmacRecordProtector(recordKey)); +} + +IOpcUaServerBuilder ua = builder.Services + .AddOpcUa() + .AddServer(o => + { + o.ApplicationName = "RedundantServer"; + o.ApplicationUri = applicationUri; + o.ProductUri = "uri:opcfoundation.org:RedundantServer"; + o.AutoAcceptUntrustedCertificates = true; + o.EndpointUrls.Add(endpointUrl); + // Transparent mode: share one certificate across replicas by pointing them + // at a common PKI store with a stable subject name. + if (!string.IsNullOrWhiteSpace(sharedSubjectName)) + { + o.SubjectName = sharedSubjectName!; + } + if (!string.IsNullOrWhiteSpace(sharedPkiRoot)) + { + o.PkiRoot = sharedPkiRoot!; + } + }) + .AddNodeManager(); + +if (activeActive) +{ + // The sample node manager gates writes on leadership; in active/active every + // replica is a writer, so make every replica a leader. + builder.Services.AddSingleton(_ => new StaticLeaderElection(true)); + + int gossipPort = int.TryParse(builder.Configuration["HA_GOSSIP_PORT"], out int gp) ? gp : 4840; + var gossipPeers = new List(); + foreach (string peer in ReadList(builder.Configuration, "HA_GOSSIP_PEERS")) + { + gossipPeers.Add(ParseEndpoint(peer)); + } + ReplicaId replicaId = ReplicaIdFromNodeId(nodeId); + + ua.UseReplicatedAddressSpace(r => + { + r.ReplicaId = replicaId; + r.UseTcpGossip(IPAddress.Any, gossipPort); + foreach (IPEndPoint peer in gossipPeers) + { + r.AddPeer(peer); + } + }) + .UseReplicatedSessions(s => + { + // Mirrored session entries gossip on a second port; peers use the + // address-space port + 1 by convention. + s.ReplicaId = replicaId; + s.UseTcpGossip(IPAddress.Any, gossipPort + 1); + foreach (IPEndPoint peer in gossipPeers) + { + s.AddPeer(new IPEndPoint(peer.Address, peer.Port + 1)); + } + s.Session.EnableFastReconnect = enableFastReconnect; + }); + builder.Services.AddSingleton(sp => + new LeaderServiceLevelProvider( + sp.GetRequiredService(), + redundancyMode)); + builder.Services.AddSingleton(sp => + new ServiceLevelStartupTask(sp.GetRequiredService())); +} +else if (redundancyMode == RedundancySupport.None) +{ + ua.AddServerServiceLevel(new ConstantServiceLevelProvider()); +} +else +{ + if (useStrongConsistency) + { + // Register the Raft-backed strongly-consistent shared store BEFORE the + // distributed address space/sessions so they compose over it. + (ulong raftId, int raftMembers, string raftBind, List raftPeers) = + ReadRaftConfig(builder.Configuration); + ua.UseRedundancyConsistency(o => + { + o.Mode = RedundancyConsistencyMode.Strong; + o.RaftConsensusFactory = _ => BuildRaftCluster(raftId, raftMembers, raftBind, raftPeers); + }); + } + + ua.UseDistributedAddressSpace(d => + { + d.UseLeaderElection = true; + d.NodeId = nodeId; + d.RedundancyMode = redundancyMode; + if (recordKey != null) + { + d.RecordProtectorFactory = _ => new AesCbcHmacRecordProtector(recordKey); + } + }) + .UseDistributedSessions(s => + { + // Mirror session state across replicas; the standby still runs the full + // ActivateSession signature check on a token-reuse reconnect. + s.EnableFastReconnect = enableFastReconnect; + }); +} + +ua.AddServerRedundancy(r => +{ + r.Mode = redundancyMode; + r.CurrentServerId = nodeId; + foreach (RedundantPeer peer in redundantPeers) + { + r.RedundantPeers.Add(peer); + } +}) +.AddRequestServerStateChange(); + +if (!string.IsNullOrWhiteSpace(balancingUrl) && redundancyMode != RedundancySupport.None) +{ + ua.UseServerLoadDirection(o => + { + o.BalancingEndpointUrl = balancingUrl!; + // Route the eligibility keyspaces (health + endpoints) to the strong store + // when Raft is configured, keeping the high-churn load weight eventual. + o.StrongEligibility = useStrongConsistency; + }); +} + +byte displayedServiceLevel = redundancyMode == RedundancySupport.None + ? ServiceLevels.Maximum + : GetDisplayedServiceLevel(activeActive, redundancyMode); +Console.WriteLine( + "HA sample node '{0}' listening at {1}; HA_MODE={2}; REDUNDANCY_MODE={3}; ServiceLevel={4} ({5}).", + nodeId, + endpointUrl, + haMode, + redundancyMode, + displayedServiceLevel, + ServiceLevels.GetSubrange(displayedServiceLevel)); +Console.WriteLine("CurrentServerId: {0}", nodeId); +Console.WriteLine("Redundant peers: {0}", FormatPeers(redundantPeers)); +if (redundancyMode is RedundancySupport.Cold or + RedundancySupport.Warm or + RedundancySupport.Hot or + RedundancySupport.HotAndMirrored) +{ + Console.WriteLine("NTRS discovery capability and FindServers peer-set provider are enabled."); +} +Console.WriteLine("RequestServerStateChange is enabled for administrator-driven Maintenance/NoData failover."); + +await builder.Build().RunAsync().ConfigureAwait(false); + +static RedundancySupport ParseRedundancyMode(string? value, RedundancySupport defaultMode) +{ + if (string.IsNullOrWhiteSpace(value)) + { + return defaultMode; + } + + return value.Trim().ToLowerInvariant() switch + { + "none" => RedundancySupport.None, + "cold" => RedundancySupport.Cold, + "warm" => RedundancySupport.Warm, + "hot" => RedundancySupport.Hot, + "hotandmirrored" or "hot-and-mirrored" or "mirrored" => RedundancySupport.HotAndMirrored, + "transparent" => RedundancySupport.Transparent, + _ => throw new FormatException( + $"Invalid REDUNDANCY_MODE '{value}'. Expected none, cold, warm, hot, hotandmirrored, or transparent.") + }; +} + +static byte GetDisplayedServiceLevel(bool activeActive, RedundancySupport redundancyMode) +{ + if (activeActive) + { + return ServiceLevels.Maximum; + } + + return redundancyMode switch + { + RedundancySupport.Cold => ServiceLevels.NoData, + RedundancySupport.Warm => ServiceLevels.DegradedMaximum, + RedundancySupport.Hot or RedundancySupport.HotAndMirrored => ServiceLevels.Maximum, + _ => ServiceLevels.DegradedMaximum + }; +} + +static ArrayOf ReadRedundantPeers(IConfiguration configuration) +{ + var peers = new List(); + foreach (string entry in ReadList(configuration, "REDUNDANT_PEERS")) + { + string[] fields = entry.Split('|', StringSplitOptions.TrimEntries); + if (fields.Length is < 1 or > 3 || string.IsNullOrWhiteSpace(fields[0])) + { + throw new FormatException( + $"Invalid REDUNDANT_PEERS entry '{entry}'; expected applicationUri|applicationName|discoveryUrl+..."); + } + + ArrayOf discoveryUrls = fields.Length >= 3 + ? new ArrayOf(fields[2].Split( + ['+'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + : []; + peers.Add(new RedundantPeer(fields[0], discoveryUrls) + { + ApplicationName = fields.Length >= 2 && !string.IsNullOrWhiteSpace(fields[1]) + ? new LocalizedText(fields[1]) + : new LocalizedText(fields[0]) + }); + } + foreach (string peerServerUri in ReadList(configuration, "peerServerUris")) + { + peers.Add(new RedundantPeer(peerServerUri, []) + { + ApplicationName = new LocalizedText(peerServerUri) + }); + } + + return new ArrayOf(peers.ToArray()); +} + +static string FormatPeers(ArrayOf peers) +{ + if (peers.IsNull || peers.Count == 0) + { + return "(none)"; + } + + var formattedPeers = new List(peers.Count); + foreach (RedundantPeer peer in peers) + { + formattedPeers.Add($"{peer.ApplicationUri} [{string.Join(", ", peer.DiscoveryUrls)}]"); + } + + return string.Join("; ", formattedPeers); +} + +static IEnumerable ReadList(IConfiguration configuration, string key) +{ + string? value = configuration[key]; + if (string.IsNullOrWhiteSpace(value)) + { + yield break; + } + + foreach (string item in value.Split( + [',', ';'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + yield return item; + } +} + +static ReplicaId ReplicaIdFromNodeId(string nodeId) +{ + // Derive a stable replica identity from the node id so it survives restarts. + byte[] hash = System.Security.Cryptography.SHA256.HashData( + System.Text.Encoding.UTF8.GetBytes(nodeId)); + return new ReplicaId(new Guid(hash.AsSpan(0, 16).ToArray())); +} + +static IPEndPoint ParseEndpoint(string hostPort) +{ + int separator = hostPort.LastIndexOf(':'); + if (separator <= 0 || separator == hostPort.Length - 1) + { + throw new FormatException($"Invalid gossip endpoint '{hostPort}'; expected host:port."); + } + + string host = hostPort[..separator]; + int port = int.Parse(hostPort[(separator + 1)..], System.Globalization.CultureInfo.InvariantCulture); + IPAddress address = IPAddress.TryParse(host, out IPAddress? ip) + ? ip + : Dns.GetHostAddresses(host)[0]; + return new IPEndPoint(address, port); +} + +static (ulong NodeId, int Members, string Bind, List Peers) ReadRaftConfig(IConfiguration configuration) +{ + ulong nodeId = ulong.TryParse(configuration["HA_RAFT_ID"], out ulong id) ? id : 1; + var peers = new List(ReadList(configuration, "HA_RAFT_PEERS")); + int members = int.TryParse(configuration["HA_RAFT_MEMBERS"], out int m) ? m : peers.Count + 1; + string bind = configuration["HA_RAFT_BIND"] ?? "tcp://0.0.0.0:6560"; + return (nodeId, members, bind, peers); +} + +static IRaftConsensus BuildRaftCluster(ulong nodeId, int members, string bind, List peers) +{ + var memberIds = new List(members); + for (int i = 1; i <= members; i++) + { + memberIds.Add((ulong)i); + } + + var transportOptions = new NanoMsgBusTransportOptions { BindAddress = bind }; + foreach (string peer in peers) + { + transportOptions.Peers.Add(peer); + } + + // The RaftCsConsensus adapter owns the node (which disposes the transport); + // MemoryStorage is volatile, so a restarted replica re-syncs from the leader. +#pragma warning disable CA2000 + var transport = new NanoMsgBusTransport(transportOptions); + var storage = new MemoryStorage(new ConfState(memberIds)); + return RaftCsConsensus.CreateCluster( + nodeId, + transport, + storage, + new RaftNodeOptions { TickInterval = TimeSpan.FromMilliseconds(50) }, + config => + { + config.ElectionTick = 10; + config.PreVote = true; + config.CheckQuorum = true; + }, + TimeSpan.FromSeconds(30)); +#pragma warning restore CA2000 +} \ No newline at end of file diff --git a/Applications/RedundantServer/Properties/AssemblyInfo.cs b/Applications/RedundantServer/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..6839e42afc --- /dev/null +++ b/Applications/RedundantServer/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +[assembly: CLSCompliant(false)] \ No newline at end of file diff --git a/Applications/RedundantServer/README.md b/Applications/RedundantServer/README.md new file mode 100644 index 0000000000..145a6421a4 --- /dev/null +++ b/Applications/RedundantServer/README.md @@ -0,0 +1,251 @@ +# High Availability Server + +This sample is a minimal Generic Host based OPC UA server that demonstrates the distributed high-availability server building blocks in `Opc.Ua.Redundancy.Server` and the active/active building blocks in `Opc.Ua.Redundancy.Server`. It registers an `AsyncCustomNodeManager`-derived node manager so the local address space participates in replication, and selects its topology with the `HA_MODE` environment variable: **active/passive** (`ap`, the default — leader election so only the active replica writes) or **active/active** (`aa` — every replica writes and converges by CRDT gossip). It drives `Server.ServiceLevel` from the replica state and publishes OPC 10000-4 §6.6 `Server.ServerRedundancy` metadata, including non-transparent discovery and manual failover. + +The bundled configuration uses the default in-memory shared key/value store. That is useful for understanding the DI wiring and for single-process experimentation, but separate OS processes do not share memory. A real multi-process or multi-host deployment needs a shared backend for `ISharedKeyValueStore` such as Redis; that backend is intentionally deferred from this small sample. + +## Run one instance + +```powershell +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62543 +``` + +Connect an OPC UA client to `opc.tcp://localhost:62543/RedundantServer` and browse to `Objects/High Availability`. The `Counter` variable is writable and is also incremented once per second while this server is leader. `ActiveReplica` shows the local active writer label. + +The startup output shows the effective OPC UA redundancy value and the ServiceLevel subrange that clients read. For example, with `HA_NODE_ID=replica-a`: + +```text +HA sample node 'replica-a' listening at opc.tcp://localhost:62543/RedundantServer; HA_MODE=ap; REDUNDANCY_MODE=Hot; ServiceLevel=255 (Healthy). +CurrentServerId: replica-a +Redundant peers: (none) +NTRS discovery capability and FindServers peer-set provider are enabled. +RequestServerStateChange is enabled for administrator-driven Maintenance/NoData failover. +``` + +## Run two instances + +Use distinct ports and stable `HA_NODE_ID` values. `REDUNDANCY_MODE` selects the OPC UA redundancy model. `HA_REDUNDANT_PEERS` provides the peer set used for `ServerUriArray`, `RedundantServerArray`, NTRS discovery registration, and `FindServers` peer results. Each peer entry is: + +```text +applicationUri|applicationName|discoveryUrl1+discoveryUrl2 +``` + +```powershell +$env:HA_NODE_ID = "replica-a" +$env:REDUNDANCY_MODE = "hot" +$env:HA_REDUNDANT_PEERS = "urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://localhost:62544/RedundantServer" +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62543 +``` + +In a second terminal: + +```powershell +$env:HA_NODE_ID = "replica-b" +$env:REDUNDANCY_MODE = "hot" +$env:HA_REDUNDANT_PEERS = "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://localhost:62543/RedundantServer" +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62544 +``` + +With the default in-memory store, each process elects against its own private store, so this two-terminal setup demonstrates endpoint, node id, service-level, and redundancy metadata configuration rather than cross-process state transfer. To turn it into a real HA pair, register a shared `ISharedKeyValueStore` that both processes can reach, as shown below. + +The second replica prints the configured redundancy value and the peer set: + +```text +HA sample node 'replica-b' listening at opc.tcp://localhost:62544/RedundantServer; HA_MODE=ap; REDUNDANCY_MODE=Hot; ServiceLevel=255 (Healthy). +CurrentServerId: replica-b +Redundant peers: urn:localhost:OPCFoundation:RedundantServer:replica-a [opc.tcp://localhost:62543/RedundantServer] +NTRS discovery capability and FindServers peer-set provider are enabled. +RequestServerStateChange is enabled for administrator-driven Maintenance/NoData failover. +``` + +## Wire up a shared store for real HA + +The single-instance defaults stay in effect until you supply a shared backend. The DI wiring below is what makes additions, removals, references, and values replicate across replicas. Replace the placeholder `RedisSharedKeyValueStore` with any `ISharedKeyValueStore` reachable by every replica (a Redis adapter is intentionally deferred from this sample): + +```csharp +builder.Services + .AddOpcUa() + .AddServer(o => + { + o.ApplicationName = "RedundantServer"; + o.ApplicationUri = applicationUri; // unique per replica + o.EndpointUrls.Add(endpointUrl); + }) + .AddNodeManager() + .UseDistributedAddressSpace(d => + { + // Shared backend reachable by every replica. The default is an + // in-process InMemorySharedKeyValueStore that is NOT shared across + // processes, so a real deployment must override it. + d.KeyValueStoreFactory = sp => new RedisSharedKeyValueStore(redisConnectionString); + + // Authenticated encryption + integrity protection for every record at + // rest. The 32-byte master key comes from a Kubernetes Secret / KMS and + // is shared by all replicas. + d.RecordProtectorFactory = _ => new AesCbcHmacRecordProtector(recordKey); + + d.UseLeaderElection = true; // lease-based leader = the single writer + d.NodeId = nodeId; // unique per replica + d.RedundancyMode = RedundancySupport.HotAndMirrored; + }) + .UseDistributedSessions(s => + { + // Mirror encrypted session state so a client can fail over to a standby + // and reconnect with a full ActivateSession (the authentication token is + // only a lookup key). The default is the safe re-auth-on-failover. + s.EnableFastReconnect = true; + }) + .AddServerRedundancy(r => + { + r.Mode = RedundancySupport.HotAndMirrored; + r.CurrentServerId = nodeId; + r.RedundantPeers.Add(new RedundantPeer( + "urn:host-b:OPCFoundation:RedundantServer:replica-b", + new ArrayOf("opc.tcp://host-b:62543/RedundantServer")) + { + ApplicationName = new LocalizedText("RedundantServer replica-b") + }); + }) + .AddRequestServerStateChange(); +``` + +### Strong consistency with Raft (no external store) + +The Redis placeholder above is one way to get a shared `ISharedKeyValueStore`. This sample also ships an in-package alternative: a strongly-consistent Raft cluster. Set `HA_CONSISTENCY=strong` and the active/passive path registers `UseRedundancyConsistency(RedundancyConsistencyMode.Strong)` backed by a multi-node RaftCs cluster over NanoMsg — the shared store, leader election, and single-use session nonce become linearizable and shared across every replica, with no Redis or other external dependency. Configure the cluster from the environment: + +| Setting | Example | Description | +| --- | --- | --- | +| `HA_CONSISTENCY` | `strong`, `eventual` | `strong` backs the shared store with a Raft cluster; `eventual` (default) keeps today's behaviour. | +| `HA_RAFT_ID` | `1` | This replica's unique Raft node id (`1..N`). | +| `HA_RAFT_MEMBERS` | `3` | Static cluster size (odd `3`/`5` for a fault-tolerant quorum). | +| `HA_RAFT_BIND` | `tcp://0.0.0.0:6560` | Local Raft transport bind address. | +| `HA_RAFT_PEERS` | `tcp://server-b:6560,tcp://server-c:6560` | The other members' Raft transport addresses. | + +See `docker-compose.raft.yml` for a runnable 3-node Raft cluster (real cross-container active/passive HA). For Kubernetes, `UseKubernetesRaftConsensus` in `Opc.Ua.Redundancy.K8s` derives the same wiring from the StatefulSet ordinal and headless-Service DNS, with a file WAL on a PersistentVolume; see `Docs/Kubernetes.md`. + +## Redundancy mode and discovery settings + +| Setting | Values | Description | +| --- | --- | --- | +| `REDUNDANCY_MODE` | `none`, `cold`, `warm`, `hot`, `hotandmirrored`, `transparent` | Selects the `Server.ServerRedundancy.RedundancySupport` value. If unset, active/passive defaults to `hot` and active/active defaults to `hotandmirrored`. | +| `HA_NODE_ID` | stable replica id | Used as this replica's `ApplicationUri` suffix (unless `HA_APPLICATION_URI` overrides it) and as `CurrentServerId` for transparent redundancy. | +| `HA_REDUNDANT_PEERS` | `applicationUri|applicationName|discoveryUrl1+discoveryUrl2`, separated by comma or semicolon | Defines the peer `RedundantPeer` set. Non-transparent modes publish peer `ApplicationUri` values in `ServerUriArray`, advertise the `NTRS` server capability, and return these peers from `FindServers`. | +| `peerServerUris` | comma/semicolon-separated application URIs | Legacy shorthand for `RedundantServerArray`; prefer `HA_REDUNDANT_PEERS` when clients must resolve peers through `FindServers`. | +| `HA_MODE` | `ap`, `aa` | Chooses active/passive shared-store replication or active/active CRDT gossip. | +| `HA_CONSISTENCY` | `strong`, `eventual` | Selects the shared-store consistency model; `strong` backs it with a Raft cluster (see *Strong consistency with Raft*). | +| `HA_FAST_RECONNECT` | `true`, `false` | Allows token-reuse reconnect for mirrored sessions. The default requires full `ActivateSession` re-authentication after failover. | +| `HA_BALANCING_URL` | a discovery URL | Enables GetEndpoints load direction (see below). A `GetEndpoints` request on this virtual/LB discovery URL is answered with the best replica's endpoints; empty (default) disables it. | +| `HA_APPLICATION_URI` | a URI | Overrides the per-replica `ApplicationUri` with a **shared** one. Required for transparent redundancy so every replica presents one logical server identity (`CreateSession` validates the client `serverUri` against it). | +| `HA_SUBJECT_NAME` | a certificate subject | Sets an explicit, stable certificate subject so replicas sharing a PKI store load one `ApplicationInstanceCertificate`. | +| `HA_PKI_ROOT` | a filesystem path | Points the certificate stores at a shared directory. Combined with `HA_SUBJECT_NAME`, replicas share one `ApplicationInstanceCertificate` (production provisions this from a secret rather than a shared volume). | + +The sample also enables `RequestServerStateChange` with `AddRequestServerStateChange()`. An administrator client can call the standard method on `Server` to request Maintenance or NoData behavior and set `Server.EstimatedReturnTime`; the server updates `Server.ServiceLevel` into the appropriate OPC UA subrange so clients back off or fail over. + +Mode-specific nodes shown by the sample: + +- `none` — leaves redundancy metadata in its single-server defaults. +- `cold`, `warm`, `hot`, `hotandmirrored` — publish `ServerUriArray` from `HA_REDUNDANT_PEERS`, retain `RedundantServerArray`, advertise `NTRS`, and return the peer `ApplicationDescription` values from `FindServers`. +- `transparent` — publishes `CurrentServerId` and `RedundantServerArray` for the transparent set. + +`Server.ServiceLevel` is driven by the sub-range-aware provider: leaders report Healthy, warm standby reports Degraded, hot/hot-and-mirrored replicas report Healthy, and cold standby reports NoData. The console prints the selected mode, server id, peer set, and initial service-level subrange at startup. + +### GetEndpoints load direction + +Setting `HA_BALANCING_URL` registers `UseServerLoadDirection(...)`: every replica publishes its health `ServiceLevel`, a load weight, and its endpoints to the shared store, and a `GetEndpoints` request that arrives on the balancing URL is answered with the best replica's endpoints — the active writer in active/passive, or the least-loaded healthy replica in active/active. Plain discovery on a replica's own URL is unaffected, and this complements (never replaces) the standard client-driven `Server.ServiceLevel` / `RedundantServerArray` selection. In a real deployment a load balancer / Kubernetes `Service` fronts the replicas at the balancing URL; the sample sets `StrongEligibility` when `HA_CONSISTENCY=strong` so the eligibility keyspaces are linearizable. It requires a shared store across replicas (use `HA_CONSISTENCY=strong` / the Raft compose). See `Docs/HighAvailability.md` for the design, conformance, and security notes. + +## Transparent redundancy (single virtual endpoint) + +The load direction above is one endpoint model; the other is **transparent redundancy**, where every replica presents *one logical server* behind *one virtual endpoint*. Unlike the non-transparent modes (the client reads `RedundantServerArray` and selects a replica), a transparent client sees a single endpoint and never chooses a replica — a load balancer routes it and mirrored session state lets it continue across a replica failure. + +To present one logical server, all replicas must share: + +- **One `ApplicationUri`** (`HA_APPLICATION_URI`). `CreateSession` validates the client-supplied `serverUri` against the server's `ApplicationUri`, so replicas that disagree would reject sessions established via discovery on a peer. +- **One `ApplicationInstanceCertificate`** (`HA_SUBJECT_NAME` + `HA_PKI_ROOT` pointing at a shared store). Under SecurityMode None the certificate is not exchanged, but secured deployments must present the same certificate from every replica. +- **Mirrored session and address-space state** (`HA_MODE=aa`, `REDUNDANCY_MODE=transparent`) so a session created on one replica can be resumed on another. + +Each replica advertises the *virtual* endpoint URL (`HA_HOST` = the load-balancer host). Because that host is a DNS name, the listener binds to all interfaces in the replica's own container while returning `opc.tcp://:62543/...` to clients, so discovery and `CreateSession` echo the single endpoint the client actually uses (`ServerBase.FilterByEndpointUrl` matches the client host to the advertised base address). A client therefore connects to one URL, and on a replica failure the load balancer routes the reconnect to the survivor, where the mirrored session resumes with a token-reuse reconnect (the full `ActivateSession` signature re-check still applies). + +`docker-compose.transparent.yml` runs this end to end: two `REDUNDANCY_MODE=transparent` replicas sharing one `ApplicationUri` and one certificate (seeded into a shared PKI volume by the first replica, then reused by the second) behind an `nginx` TCP load balancer that publishes the single virtual endpoint `opc.tcp://localhost:62543/RedundantServer`, plus a client that connects only to that endpoint. In production the shared certificate is provisioned from a Kubernetes Secret / KMS to every replica rather than self-generated (see `Docs/Kubernetes.md`), which also removes the first-start certificate race. + +## Active/passive vs active/active + +The sample selects its redundancy topology with the `HA_MODE` environment variable — `ap` (active/passive, the default) or `aa` (active/active) — and references both `Opc.Ua.Redundancy.Server` and `Opc.Ua.Redundancy.Server`: + +- **Active/passive (`HA_MODE=ap`)** — `UseDistributedAddressSpace` + `UseDistributedSessions` with leader election. One replica is the active writer; standbys hydrate from the shared store and take over on failover. Redundancy is reported as `RedundancySupport.Hot`, and clients follow `Server.ServiceLevel` / `Server.ServerRedundancy` to find the active replica. +- **Active/active (`HA_MODE=aa`)** — `UseReplicatedAddressSpace` + `UseReplicatedSessions` (CRDT gossip). Every replica accepts writes and converges without a leader; redundancy is reported as `RedundancySupport.HotAndMirrored`. Replicas gossip over TCP: set `HA_GOSSIP_PORT` (default `4840`; session entries gossip on `port + 1`) and `HA_GOSSIP_PEERS` (a comma/semicolon list of `host:port` for the other replicas' address-space gossip). A session created on one replica can be resumed on another with `ManagedSessionBuilder.WithTokenReuseFailover()` on the client. + +### Run two active/passive replicas + +Use `HA_MODE=ap` (or omit it) for the leader-elected active/passive setup. A real pair needs a shared `ISharedKeyValueStore`; with the default in-memory store this remains a wiring demonstration. + +```powershell +# replica A +$env:HA_NODE_ID = "replica-a" +$env:HA_MODE = "ap" +$env:REDUNDANCY_MODE = "hot" +$env:HA_REDUNDANT_PEERS = "urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://localhost:62544/RedundantServer" +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62543 +``` + +In a second terminal: + +```powershell +# replica B +$env:HA_NODE_ID = "replica-b" +$env:HA_MODE = "ap" +$env:REDUNDANCY_MODE = "hot" +$env:HA_REDUNDANT_PEERS = "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://localhost:62543/RedundantServer" +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62544 +``` + +The active replica writes `Counter`, the standby mirrors state through the shared store, and clients use `Server.ServiceLevel`, `Server.ServerRedundancy.RedundancySupport=Hot`, and `FindServers` peer discovery to select or fail over to the active endpoint. + +### Run two active/active replicas + +```powershell +# replica A +$env:HA_NODE_ID = "replica-a" +$env:HA_MODE = "aa" +$env:HA_GOSSIP_PORT = "4840" +$env:HA_GOSSIP_PEERS = "127.0.0.1:4842" +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62543 +``` + +In a second terminal: + +```powershell +# replica B +$env:HA_NODE_ID = "replica-b" +$env:HA_MODE = "aa" +$env:HA_GOSSIP_PORT = "4842" +$env:HA_GOSSIP_PEERS = "127.0.0.1:4840" +dotnet run --project Applications\RedundantServer\RedundantServer.csproj -- --port 62544 +``` + +Both replicas now accept writes to `Counter` and converge: a write on either endpoint propagates to the other by gossip, and the per-second increment runs on every replica, with CRDT last-writer-wins resolving the concurrent updates. Unlike the active/passive store-backed setup, active/active needs no shared `ISharedKeyValueStore` between processes — the gossip transport carries the state. + +## Docker Compose + +Two compose files run the sample as containers (the build context is the repository root): + +```powershell +# Active/active: two replicas converge by CRDT gossip (no shared store), plus a client. +docker compose -f Applications\RedundantServer\docker-compose.active-active.yml up --build + +# Active/passive: leader-election wiring demonstration, plus a client. +docker compose -f Applications\RedundantServer\docker-compose.active-passive.yml up --build + +# Strong consistency: a real 3-node RaftCs cluster (shared store across containers), plus a client. +docker compose -f Applications\RedundantServer\docker-compose.raft.yml up --build + +# GetEndpoints load direction over the Raft cluster (sets HA_BALANCING_URL on every replica). +docker compose -f Applications\RedundantServer\docker-compose.loaddirection.yml up --build + +# Transparent redundancy: two replicas as ONE logical server behind an nginx load +# balancer on a single virtual endpoint (opc.tcp://localhost:62543/RedundantServer). +docker compose -f Applications\RedundantServer\docker-compose.transparent.yml up --build +``` + +Each replica exposes its OPC UA endpoint on the host (`opc.tcp://localhost:62543/RedundantServer` and `opc.tcp://localhost:62544/RedundantServer`), and the bundled `RedundantClient` connects to one replica and follows the redundant set. Set `HA_HOST` to the reachable hostname (the compose files use the container/service name) so peers and clients can connect across the container network. The active/passive compose is a wiring demonstration only, because the default in-memory store is not shared across containers; the **Raft compose (`docker-compose.raft.yml`) is a real cross-container HA deployment** — its Raft cluster is the shared, linearizable store (see "Strong consistency with Raft" and "Wire up a shared store for real HA" above). + +For the broader design, see [HighAvailability.md](..\..\Docs\HighAvailability.md). For an environment-driven replica-set deployment, see [Kubernetes.md](..\..\Docs\Kubernetes.md). diff --git a/Applications/RedundantServer/RedundantServer.csproj b/Applications/RedundantServer/RedundantServer.csproj new file mode 100644 index 0000000000..21649b0005 --- /dev/null +++ b/Applications/RedundantServer/RedundantServer.csproj @@ -0,0 +1,31 @@ + + + net10.0 + $(CustomTestTarget) + Exe + RedundantServer + RedundantServer + OPC Foundation + Minimal .NET console OPC UA server demonstrating distributed high-availability address-space replication, service-level publishing, and server redundancy metadata. + Copyright © 2004-2025 OPC Foundation, Inc + RedundantServer + enable + false + $(NoWarn);CA1822 + true + + true + + + + + + + + + + + + diff --git a/Applications/RedundantServer/docker-compose.active-active.yml b/Applications/RedundantServer/docker-compose.active-active.yml new file mode 100644 index 0000000000..59ff8325b5 --- /dev/null +++ b/Applications/RedundantServer/docker-compose.active-active.yml @@ -0,0 +1,63 @@ +# Active/active redundant OPC UA servers (CRDT gossip) plus a managed client. +# +# Two server replicas accept writes and converge their address space over the gossip +# transport, so NO shared store is required across containers. The managed client +# connects to one replica and transparently discovers / fails over to the other +# using the server-published redundant set. +# +# The build context is the repository root. Run from the repo root: +# docker compose -f Applications/RedundantServer/docker-compose.active-active.yml up --build +# +# OPC UA endpoints exposed on the host: replica-a opc.tcp://localhost:62543/RedundantServer +# and replica-b opc.tcp://localhost:62544/RedundantServer. + +services: + server-a: + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + hostname: server-a + environment: + HA_HOST: server-a + HA_NODE_ID: replica-a + HA_MODE: aa + HA_GOSSIP_PORT: "4840" + HA_GOSSIP_PEERS: server-b:4840 + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://server-b:62543/RedundantServer" + command: ["--port", "62543"] + ports: + - "62543:62543" + + server-b: + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + hostname: server-b + environment: + HA_HOST: server-b + HA_NODE_ID: replica-b + HA_MODE: aa + HA_GOSSIP_PORT: "4840" + HA_GOSSIP_PEERS: server-a:4840 + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://server-a:62543/RedundantServer" + command: ["--port", "62543"] + ports: + - "62544:62543" + + client: + build: + context: ../.. + dockerfile: Applications/RedundantClient/Dockerfile + image: opcua-redundant-client + depends_on: + - server-a + - server-b + command: + - "--server" + - "opc.tcp://server-a:62543/RedundantServer" + - "--autoaccept" + - "--nosecurity" + - "--duration" + - "00:10:00" diff --git a/Applications/RedundantServer/docker-compose.active-passive.yml b/Applications/RedundantServer/docker-compose.active-passive.yml new file mode 100644 index 0000000000..300af6fcac --- /dev/null +++ b/Applications/RedundantServer/docker-compose.active-passive.yml @@ -0,0 +1,64 @@ +# Active/passive redundant OPC UA servers (leader election) plus a managed client. +# +# NOTE: this sample uses the default in-memory shared key/value store, which is NOT +# shared across containers. Each replica therefore elects against its own private +# store, so this compose demonstrates the redundancy wiring, endpoint and +# service-level metadata, and the client connection - not cross-container state +# transfer. For real active/passive HA, register a shared ISharedKeyValueStore +# (for example Redis) reachable by every replica; see +# Applications/RedundantServer/README.md ("Wire up a shared store for real HA"). +# +# The build context is the repository root. Run from the repo root: +# docker compose -f Applications/RedundantServer/docker-compose.active-passive.yml up --build +# +# OPC UA endpoints exposed on the host: replica-a opc.tcp://localhost:62543/RedundantServer +# and replica-b opc.tcp://localhost:62544/RedundantServer. + +services: + server-a: + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + hostname: server-a + environment: + HA_HOST: server-a + HA_NODE_ID: replica-a + HA_MODE: ap + REDUNDANCY_MODE: hot + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://server-b:62543/RedundantServer" + command: ["--port", "62543"] + ports: + - "62543:62543" + + server-b: + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + hostname: server-b + environment: + HA_HOST: server-b + HA_NODE_ID: replica-b + HA_MODE: ap + REDUNDANCY_MODE: hot + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://server-a:62543/RedundantServer" + command: ["--port", "62543"] + ports: + - "62544:62543" + + client: + build: + context: ../.. + dockerfile: Applications/RedundantClient/Dockerfile + image: opcua-redundant-client + depends_on: + - server-a + - server-b + command: + - "--server" + - "opc.tcp://server-a:62543/RedundantServer" + - "--autoaccept" + - "--nosecurity" + - "--duration" + - "00:10:00" diff --git a/Applications/RedundantServer/docker-compose.loaddirection.yml b/Applications/RedundantServer/docker-compose.loaddirection.yml new file mode 100644 index 0000000000..be198c5698 --- /dev/null +++ b/Applications/RedundantServer/docker-compose.loaddirection.yml @@ -0,0 +1,79 @@ +# GetEndpoints load direction over a strongly-consistent (Raft) redundant set. +# +# Builds on docker-compose.raft.yml: a 3-node RaftCs cluster forms the shared, linearizable +# store that the load-direction signals (each replica's health ServiceLevel, load weight, and +# endpoints) are published to and read from across containers. HA_BALANCING_URL enables the +# GetEndpoints director on every replica: a GetEndpoints request arriving on that virtual +# discovery URL is answered with the best replica's endpoints (the active writer in this +# active/passive set), while plain discovery on a replica's own URL is unaffected. +# +# In a real deployment a load balancer / Kubernetes Service fronts the replicas at +# HA_BALANCING_URL and forwards the discovery request to any replica; this compose sets the +# variable to illustrate the server-side wiring (there is no LB container here, so point a +# client at a replica's own URL to observe normal discovery, and use HA_BALANCING_URL as the +# address your LB/Service would expose). +# +# The build context is the repository root. Run from the repo root: +# docker compose -f Applications/RedundantServer/docker-compose.loaddirection.yml up --build + +x-ld-server: &ld-server + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + command: ["--port", "62543"] + +services: + server-a: + <<: *ld-server + hostname: server-a + environment: + HA_HOST: server-a + HA_NODE_ID: replica-a + HA_MODE: ap + HA_CONSISTENCY: strong + REDUNDANCY_MODE: hot + HA_BALANCING_URL: "opc.tcp://ha-lb:62543/RedundantServer" + HA_RAFT_ID: "1" + HA_RAFT_MEMBERS: "3" + HA_RAFT_BIND: "tcp://0.0.0.0:6560" + HA_RAFT_PEERS: "tcp://server-b:6560,tcp://server-c:6560" + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://server-b:62543/RedundantServer,urn:localhost:OPCFoundation:RedundantServer:replica-c|RedundantServer replica-c|opc.tcp://server-c:62543/RedundantServer" + ports: + - "62543:62543" + + server-b: + <<: *ld-server + hostname: server-b + environment: + HA_HOST: server-b + HA_NODE_ID: replica-b + HA_MODE: ap + HA_CONSISTENCY: strong + REDUNDANCY_MODE: hot + HA_BALANCING_URL: "opc.tcp://ha-lb:62543/RedundantServer" + HA_RAFT_ID: "2" + HA_RAFT_MEMBERS: "3" + HA_RAFT_BIND: "tcp://0.0.0.0:6560" + HA_RAFT_PEERS: "tcp://server-a:6560,tcp://server-c:6560" + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://server-a:62543/RedundantServer,urn:localhost:OPCFoundation:RedundantServer:replica-c|RedundantServer replica-c|opc.tcp://server-c:62543/RedundantServer" + ports: + - "62544:62543" + + server-c: + <<: *ld-server + hostname: server-c + environment: + HA_HOST: server-c + HA_NODE_ID: replica-c + HA_MODE: ap + HA_CONSISTENCY: strong + REDUNDANCY_MODE: hot + HA_BALANCING_URL: "opc.tcp://ha-lb:62543/RedundantServer" + HA_RAFT_ID: "3" + HA_RAFT_MEMBERS: "3" + HA_RAFT_BIND: "tcp://0.0.0.0:6560" + HA_RAFT_PEERS: "tcp://server-a:6560,tcp://server-b:6560" + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://server-a:62543/RedundantServer,urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://server-b:62543/RedundantServer" + ports: + - "62545:62543" diff --git a/Applications/RedundantServer/docker-compose.raft.yml b/Applications/RedundantServer/docker-compose.raft.yml new file mode 100644 index 0000000000..43914c73b1 --- /dev/null +++ b/Applications/RedundantServer/docker-compose.raft.yml @@ -0,0 +1,91 @@ +# Strongly-consistent (Raft) active/passive redundant OPC UA servers plus a managed client. +# +# Unlike docker-compose.active-passive.yml (whose default in-memory store is private to each +# container), this compose forms a real 3-node RaftCs cluster over NanoMsg: the shared key/value +# store, leader election, and single-use session nonce are linearizable and shared across every +# replica. Exactly one replica is the Raft leader (the active writer); the others are hot standbys +# that take over on leader loss. Losing one of three replicas keeps a quorum (2 of 3), so the +# cluster stays available. +# +# The build context is the repository root. Run from the repo root: +# docker compose -f Applications/RedundantServer/docker-compose.raft.yml up --build +# +# OPC UA endpoints on the host: replica-a opc.tcp://localhost:62543/RedundantServer, +# replica-b :62544, replica-c :62545. The Raft transport (port 6560) stays on the internal +# network and is addressed by container DNS. + +x-raft-server: &raft-server + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + command: ["--port", "62543"] + +services: + server-a: + <<: *raft-server + hostname: server-a + environment: + HA_HOST: server-a + HA_NODE_ID: replica-a + HA_MODE: ap + HA_CONSISTENCY: strong + REDUNDANCY_MODE: hot + HA_RAFT_ID: "1" + HA_RAFT_MEMBERS: "3" + HA_RAFT_BIND: "tcp://0.0.0.0:6560" + HA_RAFT_PEERS: "tcp://server-b:6560,tcp://server-c:6560" + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://server-b:62543/RedundantServer,urn:localhost:OPCFoundation:RedundantServer:replica-c|RedundantServer replica-c|opc.tcp://server-c:62543/RedundantServer" + ports: + - "62543:62543" + + server-b: + <<: *raft-server + hostname: server-b + environment: + HA_HOST: server-b + HA_NODE_ID: replica-b + HA_MODE: ap + HA_CONSISTENCY: strong + REDUNDANCY_MODE: hot + HA_RAFT_ID: "2" + HA_RAFT_MEMBERS: "3" + HA_RAFT_BIND: "tcp://0.0.0.0:6560" + HA_RAFT_PEERS: "tcp://server-a:6560,tcp://server-c:6560" + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://server-a:62543/RedundantServer,urn:localhost:OPCFoundation:RedundantServer:replica-c|RedundantServer replica-c|opc.tcp://server-c:62543/RedundantServer" + ports: + - "62544:62543" + + server-c: + <<: *raft-server + hostname: server-c + environment: + HA_HOST: server-c + HA_NODE_ID: replica-c + HA_MODE: ap + HA_CONSISTENCY: strong + REDUNDANCY_MODE: hot + HA_RAFT_ID: "3" + HA_RAFT_MEMBERS: "3" + HA_RAFT_BIND: "tcp://0.0.0.0:6560" + HA_RAFT_PEERS: "tcp://server-a:6560,tcp://server-b:6560" + HA_REDUNDANT_PEERS: "urn:localhost:OPCFoundation:RedundantServer:replica-a|RedundantServer replica-a|opc.tcp://server-a:62543/RedundantServer,urn:localhost:OPCFoundation:RedundantServer:replica-b|RedundantServer replica-b|opc.tcp://server-b:62543/RedundantServer" + ports: + - "62545:62543" + + client: + build: + context: ../.. + dockerfile: Applications/RedundantClient/Dockerfile + image: opcua-redundant-client + depends_on: + - server-a + - server-b + - server-c + command: + - "--server" + - "opc.tcp://server-a:62543/RedundantServer" + - "--autoaccept" + - "--nosecurity" + - "--duration" + - "00:10:00" diff --git a/Applications/RedundantServer/docker-compose.transparent.yml b/Applications/RedundantServer/docker-compose.transparent.yml new file mode 100644 index 0000000000..d8bbddc9b3 --- /dev/null +++ b/Applications/RedundantServer/docker-compose.transparent.yml @@ -0,0 +1,111 @@ +# Transparent redundancy: two replicas present ONE logical OPC UA server behind a +# single virtual endpoint. +# +# Both replicas share one ApplicationUri and one ApplicationInstanceCertificate +# (seeded into a shared PKI volume by server-a, then reused by server-b), and +# mirror address-space + session state by CRDT gossip (HA_MODE=aa, +# REDUNDANCY_MODE=transparent). An nginx TCP load balancer publishes the single +# virtual endpoint; each replica advertises that same endpoint URL (HA_HOST=lb) +# while binding to all interfaces in its own container, so discovery and +# CreateSession echo the one endpoint the client actually uses. +# +# When a replica fails, the load balancer routes the next connection to the +# survivor; because the session is mirrored, a token-reuse reconnect resumes it +# transparently on the survivor (full ActivateSession re-check still applies). +# +# The build context is the repository root. Run from the repo root: +# docker compose -f Applications/RedundantServer/docker-compose.transparent.yml up --build +# +# Single virtual OPC UA endpoint on the host: opc.tcp://localhost:62543/RedundantServer +# +# NOTE: this sample connects with SecurityMode None, so the per-replica cert race +# on first start is benign here. A SECURED transparent deployment must PRE-PROVISION +# the single shared ApplicationInstanceCertificate to every replica (for example a +# Kubernetes Secret / KMS), never let each replica self-generate one. + +services: + server-a: + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + hostname: server-a + environment: + # Advertise the shared virtual endpoint (the load balancer host). The host + # is a DNS name, so the listener binds to all interfaces in this container + # while returning opc.tcp://lb:62543/... to clients. + HA_HOST: lb + HA_NODE_ID: replica-a + HA_MODE: aa + REDUNDANCY_MODE: transparent + # Shared logical identity: every replica MUST use the same ApplicationUri + # (CreateSession validates the client-supplied serverUri against it) and the + # same certificate subject. + HA_APPLICATION_URI: urn:OPCFoundation:RedundantServer:transparent + HA_SUBJECT_NAME: "CN=RedundantServer, O=OPC Foundation, DC=lb" + HA_PKI_ROOT: /shared/pki + HA_GOSSIP_PORT: "4840" + HA_GOSSIP_PEERS: server-b:4840 + command: ["--port", "62543"] + volumes: + - shared-pki:/shared/pki + # server-a seeds the shared certificate; report healthy once the PKI store is + # populated so server-b starts afterwards and reuses the same certificate. + healthcheck: + test: ["CMD-SHELL", "test -n \"$(ls -A /shared/pki 2>/dev/null)\""] + interval: 3s + timeout: 2s + retries: 20 + + server-b: + build: + context: ../.. + dockerfile: Applications/RedundantServer/Dockerfile + image: opcua-redundant-server + hostname: server-b + environment: + HA_HOST: lb + HA_NODE_ID: replica-b + HA_MODE: aa + REDUNDANCY_MODE: transparent + HA_APPLICATION_URI: urn:OPCFoundation:RedundantServer:transparent + HA_SUBJECT_NAME: "CN=RedundantServer, O=OPC Foundation, DC=lb" + HA_PKI_ROOT: /shared/pki + HA_GOSSIP_PORT: "4840" + HA_GOSSIP_PEERS: server-a:4840 + command: ["--port", "62543"] + volumes: + - shared-pki:/shared/pki + depends_on: + server-a: + condition: service_healthy + + lb: + image: nginx:1.27-alpine + hostname: lb + depends_on: + - server-a + - server-b + volumes: + - ./nginx.transparent.conf:/etc/nginx/nginx.conf:ro + ports: + # The single virtual OPC UA endpoint exposed to clients on the host. + - "62543:62543" + + client: + build: + context: ../.. + dockerfile: Applications/RedundantClient/Dockerfile + image: opcua-redundant-client + depends_on: + - lb + command: + - "--server" + - "opc.tcp://lb:62543/RedundantServer" + - "--autoaccept" + - "--nosecurity" + - "--duration" + - "00:10:00" + +volumes: + shared-pki: diff --git a/Applications/RedundantServer/nginx.transparent.conf b/Applications/RedundantServer/nginx.transparent.conf new file mode 100644 index 0000000000..4d15ef5d1a --- /dev/null +++ b/Applications/RedundantServer/nginx.transparent.conf @@ -0,0 +1,25 @@ +# nginx TCP (stream) load balancer for the transparent-redundancy sample. +# +# Fronts the two replicas with ONE virtual OPC UA endpoint on port 62543 and +# round-robins new TCP connections across them. On a connection failure nginx +# marks the replica down (max_fails/fail_timeout) and routes the next connection +# to the survivor, so a client reconnecting after a replica fails lands on the +# healthy replica - and, because session state is mirrored, resumes transparently. +events {} + +stream { + upstream opcua_replicas { + server server-a:62543 max_fails=1 fail_timeout=5s; + server server-b:62543 max_fails=1 fail_timeout=5s; + } + + server { + listen 62543; + proxy_pass opcua_replicas; + proxy_connect_timeout 2s; + # OPC UA channels are long-lived; do not time out an idle secure channel. + proxy_timeout 10m; + # On a connect/read failure, transparently retry the next replica. + proxy_next_upstream on; + } +} diff --git a/Directory.Build.targets b/Directory.Build.targets index af3baf91bc..b8d89d8bf4 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -44,6 +44,10 @@ false false false + + v4.0.30319 diff --git a/Directory.Packages.props b/Directory.Packages.props index 8dcf49f2d1..6c7ff8a416 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,6 +13,12 @@ + + + + + + @@ -30,8 +36,8 @@ - - + + @@ -109,11 +115,11 @@ - + - + + net8.0;net9.0;net10.0 + $(CustomTestTarget) + $(CustomTestTarget) + true + $(PackagePrefix).Opc.Ua.Redundancy.K8s + $(AssemblyPrefix).Server.Redundancy.K8s + Opc.Ua.Redundancy.K8s + OPC UA distributed server Kubernetes integration for leader election, peer discovery, and readiness. + PackageReference + true + NugetREADME.md + true + enable + true + true + + + $(PackageId).Debug + + + + + + + + + + + diff --git a/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/IKubernetesPeerDiscovery.cs b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/IKubernetesPeerDiscovery.cs new file mode 100644 index 0000000000..d2b7c0a803 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/IKubernetesPeerDiscovery.cs @@ -0,0 +1,58 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.K8s +{ + /// + /// Extension beyond OPC 10000-4 §6.6: discovers OPC UA peer ServerUris from Kubernetes EndpointSlices. + /// + public interface IKubernetesPeerDiscovery + { + /// + /// Raised when the discovered peer ServerUri set changes. + /// + event Action>? PeerServerUrisChanged; + + /// + /// Gets the last discovered peer ServerUris. + /// + ArrayOf PeerServerUris { get; } + + /// + /// Refreshes peer discovery once. + /// + /// Cancellation token. + /// The discovered peer ServerUris. + ValueTask> RefreshAsync(CancellationToken ct = default); + } +} \ No newline at end of file diff --git a/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscovery.cs b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscovery.cs new file mode 100644 index 0000000000..f58c041216 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscovery.cs @@ -0,0 +1,187 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy.Server; + +namespace Opc.Ua.Redundancy.K8s +{ + /// + /// Extension beyond OPC 10000-4 §6.6: EndpointSlice-backed Kubernetes peer discovery for + /// non-transparent RedundantServerSet ServerUris. + /// + public sealed class KubernetesPeerDiscovery : IKubernetesPeerDiscovery + { + /// + /// Creates a Kubernetes peer discovery service using the in-cluster API client. + /// + /// The peer discovery options. + public KubernetesPeerDiscovery(KubernetesPeerDiscoveryOptions options) + : this(CreateApiClient(options), options) + { + } + + private static IKubernetesApiClient CreateApiClient(KubernetesPeerDiscoveryOptions options) + { + return KubernetesApiClientFactory.Create( + (options ?? throw new ArgumentNullException(nameof(options))).Kubernetes); + } + + internal KubernetesPeerDiscovery(IKubernetesApiClient apiClient, KubernetesPeerDiscoveryOptions options) + { + m_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_namespace = KubernetesApiClientFactory.ResolveNamespace(m_options.Kubernetes, m_apiClient); + } + + /// + public event Action>? PeerServerUrisChanged; + + /// + public ArrayOf PeerServerUris + { + get + { + lock (m_lock) + { + return m_peerServerUris; + } + } + } + + /// + public async ValueTask> RefreshAsync(CancellationToken ct = default) + { + if (!m_apiClient.IsInCluster) + { + SetPeers(ArrayOf.Empty); + return PeerServerUris; + } + + KubernetesEndpointSliceList slices = await m_apiClient + .ListEndpointSlicesAsync(m_namespace, m_options.ServiceName, ct) + .ConfigureAwait(false); + ArrayOf peers = ToPeerUris(slices, m_options); + SetPeers(peers); + return peers; + } + + /// + /// Copies discovered peer ServerUris into the base redundancy options. + /// + /// The redundancy options to populate. + public void Populate(ServerRedundancyOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + options.PeerServerUris.Clear(); + foreach (string uri in PeerServerUris.Span) + { + options.PeerServerUris.Add(uri); + } + } + + internal static ArrayOf ToPeerUris( + KubernetesEndpointSliceList slices, + KubernetesPeerDiscoveryOptions options) + { + if (slices == null) + { + throw new ArgumentNullException(nameof(slices)); + } + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + var uris = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (KubernetesEndpointSlice slice in slices.Items) + { + int port = SelectPort(slice, options); + foreach (KubernetesEndpoint endpoint in slice.Endpoints) + { + if (endpoint.Conditions?.Ready == false) + { + continue; + } + + foreach (string address in endpoint.Addresses) + { + if (string.IsNullOrWhiteSpace(address) || + string.Equals(address, options.LocalAddress, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + uris.Add(string.Create( + CultureInfo.InvariantCulture, + $"{options.UriScheme}://{address}:{port}")); + } + } + } + + return new ArrayOf(uris.ToArray().AsMemory()); + } + + private static int SelectPort(KubernetesEndpointSlice slice, KubernetesPeerDiscoveryOptions options) + { + KubernetesEndpointPort? named = slice.Ports.FirstOrDefault(port => + string.Equals(port.Name, options.PortName, StringComparison.OrdinalIgnoreCase)); + return named?.Port ?? slice.Ports.FirstOrDefault(port => port.Port.HasValue)?.Port ?? options.Port; + } + + private void SetPeers(ArrayOf peers) + { + bool changed; + lock (m_lock) + { + changed = !m_peerServerUris.Span.SequenceEqual(peers.Span); + m_peerServerUris = peers; + } + if (changed) + { + PeerServerUrisChanged?.Invoke(peers); + } + } + + private readonly IKubernetesApiClient m_apiClient; + private readonly KubernetesPeerDiscoveryOptions m_options; + private readonly string m_namespace; + private readonly Lock m_lock = new(); + private ArrayOf m_peerServerUris = ArrayOf.Empty; + } +} \ No newline at end of file diff --git a/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryOptions.cs b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryOptions.cs new file mode 100644 index 0000000000..e25fdae65b --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryOptions.cs @@ -0,0 +1,74 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Redundancy.K8s +{ + /// + /// Extension beyond OPC 10000-4 §6.6: options for Kubernetes EndpointSlice peer discovery. + /// + public sealed class KubernetesPeerDiscoveryOptions + { + /// + /// Gets the shared Kubernetes client options. + /// + public KubernetesServerOptions Kubernetes { get; } = new(); + + /// + /// Gets or sets the headless Service name whose EndpointSlices contain OPC UA peers. + /// + public string ServiceName { get; set; } = "opcua-ha-headless"; + + /// + /// Gets or sets the EndpointSlice port name to prefer. + /// + public string? PortName { get; set; } = "opcua-tcp"; + + /// + /// Gets or sets the OPC UA URI scheme used for discovered peers. + /// + public string UriScheme { get; set; } = "opc.tcp"; + + /// + /// Gets or sets the fallback OPC UA port when EndpointSlices do not name one. + /// + public int Port { get; set; } = 4840; + + /// + /// Gets or sets this pod DNS name or address to exclude from the discovered peer list. + /// + public string? LocalAddress { get; set; } + + /// + /// Gets or sets how often the optional background refresh loop polls EndpointSlices. + /// + public TimeSpan RefreshInterval { get; set; } = TimeSpan.FromSeconds(15); + } +} \ No newline at end of file diff --git a/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryStartupTask.cs b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryStartupTask.cs new file mode 100644 index 0000000000..f4c0d8d5d8 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryStartupTask.cs @@ -0,0 +1,92 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; + +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.K8s +{ + internal sealed class KubernetesPeerDiscoveryStartupTask : IServerStartupTask, IAsyncDisposable + { + public KubernetesPeerDiscoveryStartupTask( + IKubernetesPeerDiscovery discovery, + KubernetesPeerDiscoveryOptions options) + { + m_discovery = discovery ?? throw new ArgumentNullException(nameof(discovery)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + m_loop = Task.Run(() => RefreshLoopAsync(m_cts.Token), CancellationToken.None); + return default; + } + + public async ValueTask DisposeAsync() + { + m_cts.Cancel(); + if (m_loop != null) + { + try + { + await m_loop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + m_cts.Dispose(); + } + + private async Task RefreshLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + await m_discovery.RefreshAsync(ct).ConfigureAwait(false); + await Task.Delay(m_options.RefreshInterval, ct).ConfigureAwait(false); + } + } + + private readonly IKubernetesPeerDiscovery m_discovery; + private readonly KubernetesPeerDiscoveryOptions m_options; + private readonly CancellationTokenSource m_cts = new(); + private Task? m_loop; + } +} \ No newline at end of file diff --git a/Libraries/Opc.Ua.Redundancy.K8s/Properties/AssemblyInfo.cs b/Libraries/Opc.Ua.Redundancy.K8s/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..4c76d0af81 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.K8s/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Runtime.CompilerServices; + +[assembly: CLSCompliant(false)] +[assembly: InternalsVisibleTo("Opc.Ua.Aot.Tests")] +[assembly: InternalsVisibleTo("Opc.Ua.Redundancy.K8s.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] \ No newline at end of file diff --git a/Libraries/Opc.Ua.Redundancy.Server/DistributedServerBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/DistributedServerBuilderExtensions.cs new file mode 100644 index 0000000000..2e4e81b28d --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/DistributedServerBuilderExtensions.cs @@ -0,0 +1,194 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Fluent registration of OPC 10000-4 §6.6 redundancy and value-add distributed-server extensions on the + /// . + /// + public static class DistributedServerBuilderExtensions + { + /// + /// Drives the live Server.ServiceLevel node from the supplied + /// (e.g. + /// for non-transparent Failover). Use a + /// to report a fixed level. See OPC 10000-4 §6.6.2.4.2 and + /// §6.6.2.4.3. + /// + /// The server builder. + /// The service-level source. + public static IOpcUaServerBuilder AddServerServiceLevel( + this IOpcUaServerBuilder builder, + IServiceLevelProvider serviceLevelProvider) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + if (serviceLevelProvider == null) + { + throw new ArgumentNullException(nameof(serviceLevelProvider)); + } + + builder.Services.AddSingleton(serviceLevelProvider); + builder.Services.AddSingleton( + sp => new ServiceLevelStartupTask(sp.GetRequiredService())); + return builder; + } + + /// + /// Extension beyond OPC 10000-4 §6.6: registers dependency-injection building blocks for a distributed + /// AddressSpace used by a RedundantServerSet. + /// + /// + /// This registers the shared key/value store, leader election, + /// service-level provider, and service-level startup task. Node-state + /// stores and per-node-manager synchronizers are wired later at server + /// startup, when the server message context is available. + /// + /// The server builder. + /// Optional distributed address-space options. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseDistributedAddressSpace( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new DistributedAddressSpaceOptions(); + configure?.Invoke(options); + + builder.Services.TryAddSingleton(sp => + options.KeyValueStoreFactory?.Invoke(sp) ?? new InMemorySharedKeyValueStore()); + + if (options.RecordProtectorFactory != null) + { + builder.Services.TryAddSingleton( + sp => options.RecordProtectorFactory(sp)); + } + + builder.Services.TryAddSingleton(sp => + options.UseLeaderElection + ? new SharedStoreLeaseElection( + sp.GetRequiredService(), + options.LeaseKey, + options.NodeId, + options.LeaseDuration, + options.RenewInterval) + : new StaticLeaderElection(true)); + + builder.Services.TryAddSingleton(sp => + new LeaderServiceLevelProvider( + sp.GetRequiredService(), + options.RedundancyMode, + options.ServiceLevelLoadMetric, + options.HealthServiceLevel)); + + // The service-level startup task updates Server.ServiceLevel from + // the provider. + builder.Services.AddSingleton(sp => + new ServiceLevelStartupTask(sp.GetRequiredService())); + + // The address-space startup task builds the node-state store with + // the server message context (only available at startup), starts + // leader election, and attaches a synchronizer to every node + // manager that opts in via ILocalAddressSpaceSource. + builder.Services.AddSingleton(sp => + new DistributedAddressSpaceStartupTask( + sp.GetRequiredService(), + sp.GetRequiredService(), + RecordProtectionGuard.ResolveProtectorOrThrow(sp))); + + return builder; + } + + /// + /// Registers a distributed session manager so a client can Failover to + /// a standby replica and reconnect by re-running ActivateSession + /// (OPC UA HotAndMirrored fast reconnect, OPC 10000-4 §6.6.2.4.5.5). + /// + /// + /// + /// This shares the same and + /// as + /// when composed with it; used + /// on its own it falls back to an in-memory store and a no-op protector + /// (suitable for development only). The mirrored session record is + /// encrypted at rest and the server nonce is single-use across the + /// replica set. + /// + /// + /// The safe default is re-authentication on failover. Set + /// to opt + /// into the mirrored fast reconnect; even then a reconnect performs the + /// full ActivateSession client-signature validation — the token + /// is never an authenticator on its own. See + /// Docs/HighAvailability.md. + /// + /// + /// The server builder. + /// Optional distributed session options. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseDistributedSessions( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new DistributedSessionOptions(); + configure?.Invoke(options); + + builder.Services.TryAddSingleton(_ => new InMemorySharedKeyValueStore()); + + builder.Services.TryAddSingleton(sp => + new DistributedSessionManagerFactory( + sp.GetRequiredService(), + RecordProtectionGuard.ResolveProtectorOrThrow(sp), + options)); + + return builder; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/IStrongKeyspaceProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/IStrongKeyspaceProvider.cs new file mode 100644 index 0000000000..991ead6fcc --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/IStrongKeyspaceProvider.cs @@ -0,0 +1,44 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Contributes additional key prefixes that a feature needs routed to the linearizable (Raft) store when the + /// RedundantServerSet runs in the eventual/hybrid consistency mode. UseRedundancyConsistency + /// aggregates every registered provider with the configured StrongKeyPrefixes. + /// + public interface IStrongKeyspaceProvider + { + /// + /// Returns the key prefixes this feature requires to be linearizable. + /// + ArrayOf GetStrongKeyPrefixes(); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/NugetREADME.md b/Libraries/Opc.Ua.Redundancy.Server/NugetREADME.md new file mode 100644 index 0000000000..5104a07799 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/NugetREADME.md @@ -0,0 +1,35 @@ +# OPC UA .NET Standard — Distributed / High-Availability Server + +`OPCFoundation.NetStandard.Opc.Ua.Redundancy.Server` adds the optional distributed building blocks that let an `OPCFoundation.NetStandard.Opc.Ua.Server` server run as a redundant replica set (active/passive or active/active) while sharing its address space and — optionally — its session state across replicas. + +## Overview + +The core server library stays a self-contained, single-instance server. This package layers the distributed concerns on top through dependency injection so the in-memory, single-instance path is unchanged when the package is not used: + +- A shared, integrity-protected key/value backend (`ISharedKeyValueStore`) that mirrors node additions, removals, references, and values across replicas. +- Leader election (`ILeaderElection`) for the shared-read / leader-write redundancy model, surfaced to clients through the standard OPC UA `ServiceLevel` and redundancy nodes. +- Opt-in mirrors for HotAndMirrored/Transparent failover: encrypted session state with single-use nonce validation, subscription definitions, retransmission queues for `Republish`, best-effort continuation-point envelopes, and deterministic EventIds when an `IEventIdProvider` such as `DeterministicEventIdProvider` is configured. + +## Getting started + +Wire the distributed address space and (optionally) shared sessions through the fluent DI surface: + +```csharp +services.AddOpcUa() + .AddServer(...) + .UseDistributedAddressSpace(distributed => + { + distributed.UseSharedKeyValueStore(new InMemorySharedKeyValueStore()); + }) + .UseDistributedSessions(); +``` + +The single-instance defaults remain in effect until a shared store is supplied, so the same server binary runs stand-alone or as part of a replica set. + +## Target frameworks + +`net472`, `net48`, `netstandard2.1`, `net8.0`, `net9.0`, `net10.0`. + +## Additional documentation + +See the [High Availability guide](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/HighAvailability.md) for the OPC UA redundancy mapping and the [Kubernetes deployment guide](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/Docs/Kubernetes.md) for running the server as a replica set. diff --git a/Libraries/Opc.Ua.Redundancy.Server/Opc.Ua.Redundancy.Server.csproj b/Libraries/Opc.Ua.Redundancy.Server/Opc.Ua.Redundancy.Server.csproj new file mode 100644 index 0000000000..37b7fc68d1 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Opc.Ua.Redundancy.Server.csproj @@ -0,0 +1,35 @@ + + + $(LibTargetFrameworks) + $(PackagePrefix).Opc.Ua.Redundancy.Server + $(AssemblyPrefix).Redundancy.Server + Opc.Ua.Redundancy.Server + OPC UA Distributed / High-Availability Server Class Library + PackageReference + true + NugetREADME.md + true + enable + true + true + + + + + + + + + + + $(PackageId).Debug + + + + + + + + + + diff --git a/Libraries/Opc.Ua.Redundancy.Server/Properties/AssemblyInfo.cs b/Libraries/Opc.Ua.Redundancy.Server/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..6f476448c4 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Runtime.CompilerServices; + +[assembly: CLSCompliant(false)] +[assembly: InternalsVisibleTo("Opc.Ua.Aot.Tests")] +[assembly: InternalsVisibleTo("Opc.Ua.Redundancy.Server")] diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ConfiguredRedundantServerSetProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ConfiguredRedundantServerSetProvider.cs new file mode 100644 index 0000000000..04d0abd9cd --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ConfiguredRedundantServerSetProvider.cs @@ -0,0 +1,83 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Builds non-transparent RedundantServerSet peer entries from + /// for FindServers (OPC 10000-4 §6.6.2.4.5.1). + /// + public sealed class ConfiguredRedundantServerSetProvider : IRedundantServerSetProvider + { + /// + /// Creates the provider. + /// + /// The redundancy options. + public ConfiguredRedundantServerSetProvider(ServerRedundancyOptions options) + { + m_options = options ?? throw new ArgumentNullException(nameof(options)); + } + + /// + public ArrayOf GetRedundantServerSet() + { + if (!m_options.IsNonTransparentMode) + { + return []; + } + + var descriptions = new List(m_options.RedundantPeers.Count); + foreach (RedundantPeer peer in m_options.RedundantPeers) + { + if (string.IsNullOrEmpty(peer.ApplicationUri) || peer.DiscoveryUrls.IsNull) + { + continue; + } + + descriptions.Add(new ApplicationDescription + { + ApplicationUri = peer.ApplicationUri, + ApplicationName = peer.ApplicationName.IsNull + ? new LocalizedText(peer.ApplicationUri) + : peer.ApplicationName, + ApplicationType = ApplicationType.Server, + DiscoveryUrls = peer.DiscoveryUrls + }); + } + + return new ArrayOf(descriptions.ToArray()); + } + + private readonly ServerRedundancyOptions m_options; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ConstantServiceLevelProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ConstantServiceLevelProvider.cs new file mode 100644 index 0000000000..a3b8f6eef4 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ConstantServiceLevelProvider.cs @@ -0,0 +1,67 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// An that reports a fixed service + /// level (255 by default). This preserves the historical single-instance + /// behavior and is the default when no redundancy is configured. + /// + public sealed class ConstantServiceLevelProvider : IServiceLevelProvider + { + /// + /// Creates a provider reporting a fixed service level. + /// + /// The fixed service level (default 255). + public ConstantServiceLevelProvider(byte level = ServiceLevels.Maximum) + { + m_level = level; + } + + /// + public event Action? ServiceLevelChanged + { + // The level never changes, so there is nothing to raise. + add { } + remove { } + } + + /// + public byte GetServiceLevel() + { + return m_level; + } + + private readonly byte m_level; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/IServiceLevelController.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/IServiceLevelController.cs new file mode 100644 index 0000000000..45ce2706d2 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/IServiceLevelController.cs @@ -0,0 +1,44 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Allows OPC 10000-4 §6.6.5 administrative Server state changes to publish a new ServiceLevel through the + /// active provider. + /// + public interface IServiceLevelController + { + /// + /// Sets the ServiceLevel and raises the provider change notification. + /// + /// The service level to publish. + void SetServiceLevel(byte serviceLevel); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/IServiceLevelProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/IServiceLevelProvider.cs new file mode 100644 index 0000000000..337ceb00be --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/IServiceLevelProvider.cs @@ -0,0 +1,60 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Computes the value of the Server's OPC UA ServiceLevel + /// variable (0–255). In a RedundantServerSet, Clients select the Server + /// reporting the highest ServiceLevel (OPC 10000-4 §6.6.2.4.2 and §6.6.2.4.3), so a healthy + /// active leader reports the maximum and standbys report a lower value. + /// + /// + /// Wire this into the server so the Server.ServiceLevel node is + /// initialized from and updated whenever + /// fires. The default + /// () reports a fixed 255, + /// preserving single-instance behavior. + /// + public interface IServiceLevelProvider + { + /// + /// The current ServiceLevel (Table 105: Maintenance 0, NoData 1, Degraded 2-199, Healthy 200-255). + /// + byte GetServiceLevel(); + + /// + /// Raised with the new service level whenever it changes. + /// + event Action? ServiceLevelChanged; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LeaderServiceLevelProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LeaderServiceLevelProvider.cs new file mode 100644 index 0000000000..e30d251703 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LeaderServiceLevelProvider.cs @@ -0,0 +1,164 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// An driven by leadership: the + /// leader reports the highest level so Clients prefer it; standbys report + /// a lower level. This is how non-transparent server Failover is advertised + /// through the OPC 10000-4 §6.6.2.4.2 ServiceLevel mechanism. + /// + public sealed class LeaderServiceLevelProvider : IServiceLevelProvider, IServiceLevelController, IDisposable + { + /// + /// Creates a leadership-driven service-level provider. + /// + /// The leader election to follow. + /// The configured redundancy failover mode. + /// Optional connected-client load function. + /// Optional health-derived maximum service level. + public LeaderServiceLevelProvider( + ILeaderElection election, + RedundancySupport failoverMode = RedundancySupport.Warm, + Func? getConnectedClientCount = null, + Func? getHealthServiceLevel = null) + { + m_election = election ?? throw new ArgumentNullException(nameof(election)); + m_failoverMode = failoverMode; + m_getConnectedClientCount = getConnectedClientCount; + m_getHealthServiceLevel = getHealthServiceLevel; + m_election.LeadershipChanged += OnLeadershipChanged; + } + + /// + /// Creates a leadership-driven service-level provider with explicit + /// levels. Use the mode-based overload for OPC 10000-4 subrange-aware + /// defaults. + /// + /// The leader election to follow. + /// Level reported while leader. + /// Level reported while standby. + public LeaderServiceLevelProvider( + ILeaderElection election, + byte leaderLevel, + byte standbyLevel) + : this(election, RedundancySupport.Warm) + { + m_leaderLevel = leaderLevel; + m_standbyLevel = standbyLevel; + } + + /// + public event Action? ServiceLevelChanged; + + /// + public byte GetServiceLevel() + { + int manualServiceLevel = Volatile.Read(ref m_manualServiceLevel); + if (manualServiceLevel >= 0) + { + return (byte)manualServiceLevel; + } + + byte level = m_election.IsLeader ? m_leaderLevel : GetStandbyServiceLevel(); + if (m_getHealthServiceLevel != null) + { + level = Math.Min(level, m_getHealthServiceLevel()); + } + + return ApplyHealthyLoadBalancing(level); + } + + /// + public void SetServiceLevel(byte serviceLevel) + { + Volatile.Write(ref m_manualServiceLevel, serviceLevel); + ServiceLevelChanged?.Invoke(serviceLevel); + } + + /// + /// Stops following the leader election. + /// + public void Dispose() + { + m_election.LeadershipChanged -= OnLeadershipChanged; + } + + private void OnLeadershipChanged(bool isLeader) + { + Volatile.Write(ref m_manualServiceLevel, kNoManualServiceLevel); + ServiceLevelChanged?.Invoke(GetServiceLevel()); + } + + private byte GetStandbyServiceLevel() + { + if (m_standbyLevel.HasValue) + { + return m_standbyLevel.GetValueOrDefault(); + } + + return m_failoverMode switch + { + RedundancySupport.Cold => ServiceLevels.NoData, + RedundancySupport.Hot => ServiceLevels.Maximum, + RedundancySupport.HotAndMirrored => ServiceLevels.Maximum, + _ => ServiceLevels.DegradedMaximum + }; + } + + private byte ApplyHealthyLoadBalancing(byte level) + { + if (m_getConnectedClientCount == null || !ServiceLevels.IsHealthy(level)) + { + return level; + } + + uint clientCount = m_getConnectedClientCount(); + byte maximumDecrement = (byte)(level - ServiceLevels.HealthyMinimum); + byte decrement = clientCount > maximumDecrement ? maximumDecrement : (byte)clientCount; + return (byte)(level - decrement); + } + + private readonly ILeaderElection m_election; + private readonly RedundancySupport m_failoverMode; + private readonly Func? m_getConnectedClientCount; + private readonly Func? m_getHealthServiceLevel; + private byte m_leaderLevel = ServiceLevels.Maximum; + private byte? m_standbyLevel; + private int m_manualServiceLevel = kNoManualServiceLevel; + + private const int kNoManualServiceLevel = -1; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/BandedServerDirectionPolicy.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/BandedServerDirectionPolicy.cs new file mode 100644 index 0000000000..dfd365c184 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/BandedServerDirectionPolicy.cs @@ -0,0 +1,155 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Default . Ranks members by a health ServiceLevel tier (Healthy, + /// Degraded, NoData, Maintenance — optionally sub-banded within Healthy), keeps only the top tier (including the + /// local Server), then chooses the least-loaded member using the separate load weight, breaking equal-load bands + /// at random. A stale or unknown peer view fails safe to the local Server. + /// + public sealed class BandedServerDirectionPolicy : IServerDirectionPolicy + { + /// + /// Creates the policy. + /// + /// The peer direction view. + /// The load-direction options (band sizes). + /// + /// Chooses an index in [0, count) for random tie-breaking; must be safe for concurrent use. + /// + public BandedServerDirectionPolicy( + IPeerDirectionView view, + LoadDirectionOptions options, + Func chooseIndex) + { + m_view = view ?? throw new ArgumentNullException(nameof(view)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_chooseIndex = chooseIndex ?? throw new ArgumentNullException(nameof(chooseIndex)); + } + + /// + public async ValueTask SelectTargetServerUriAsync( + string localServerUri, + byte localServiceLevel, + byte localLoadWeight, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(localServerUri)) + { + return null; + } + + ArrayOf peers; + try + { + peers = await m_view.GetPeersAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Fail safe to self when the peer view cannot be read. + return null; + } + + int loadBandSize = Math.Max(1, m_options.LoadBandSize); + int healthSubBandSize = m_options.HealthSubBandSize; + + var members = new List + { + new(localServerUri, HealthRank(localServiceLevel, healthSubBandSize), localLoadWeight / loadBandSize, true) + }; + int topRank = members[0].HealthRank; + + foreach (PeerDirectionRecord peer in peers) + { + if (string.Equals(peer.ServerUri, localServerUri, StringComparison.Ordinal)) + { + continue; + } + int rank = HealthRank(peer.ServiceLevel, healthSubBandSize); + int loadBand = peer.LoadKnown ? peer.LoadWeight / loadBandSize : int.MaxValue; + members.Add(new Member(peer.ServerUri, rank, loadBand, false)); + if (rank > topRank) + { + topRank = rank; + } + } + + List eligible = members.Where(m => m.HealthRank == topRank).ToList(); + int minLoadBand = eligible.Min(m => m.LoadBand); + List leastLoaded = eligible.Where(m => m.LoadBand == minLoadBand).ToList(); + + Member chosen = leastLoaded.Count == 1 + ? leastLoaded[0] + : leastLoaded[BoundedIndex(leastLoaded.Count)]; + + return chosen.IsSelf ? null : chosen.ServerUri; + } + + private int BoundedIndex(int count) + { + int index = m_chooseIndex(count); + return index < 0 || index >= count ? 0 : index; + } + + private static int HealthRank(byte level, int healthSubBandSize) + { + if (ServiceLevels.IsHealthy(level)) + { + int sub = healthSubBandSize > 0 + ? (level - ServiceLevels.HealthyMinimum) / healthSubBandSize + : 0; + return HealthyBaseRank + sub; + } + if (level >= DegradedMinimum) + { + return DegradedRank; + } + // NoData (1) ranks above Maintenance (0). + return level; + } + + private const int HealthyBaseRank = 3; + private const int DegradedRank = 2; + private const byte DegradedMinimum = 2; + + private readonly IPeerDirectionView m_view; + private readonly LoadDirectionOptions m_options; + private readonly Func m_chooseIndex; + + private readonly record struct Member(string ServerUri, int HealthRank, int LoadBand, bool IsSelf); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ConstantLoadWeightProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ConstantLoadWeightProvider.cs new file mode 100644 index 0000000000..43d6889a56 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ConstantLoadWeightProvider.cs @@ -0,0 +1,65 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Default that reports a fixed load weight (0 by default). With the constant + /// provider, load direction degenerates to random selection among peers tied at the highest health + /// ServiceLevel, preserving simple behaviour when no load metric is wired. + /// + public sealed class ConstantLoadWeightProvider : ILoadWeightProvider + { + /// + /// Creates the provider. + /// + /// The fixed load weight to report (default 0 = idle). + public ConstantLoadWeightProvider(byte loadWeight = 0) + { + m_loadWeight = loadWeight; + } + + /// + public byte GetLoadWeight() + { + return m_loadWeight; + } + + /// + public event Action? LoadWeightChanged + { + add { } + remove { } + } + + private readonly byte m_loadWeight; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ILoadWeightProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ILoadWeightProvider.cs new file mode 100644 index 0000000000..35ceeaf08d --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ILoadWeightProvider.cs @@ -0,0 +1,57 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: reports the local Server's current load weight (0 = idle, + /// 255 = fully loaded) used only to break ties among peers already tied at the highest health + /// ServiceLevel when directing a Client to the best Server in a RedundantServerSet. + /// + /// + /// The load weight is deliberately a separate signal from ServiceLevel: ServiceLevel keeps + /// its OPC UA meaning (health/eligibility and Failover), while a stale or missing load weight only affects + /// tie-breaking and never eligibility. The default reports a fixed 0, + /// which reduces load direction to random selection among equally-healthy peers. + /// + public interface ILoadWeightProvider + { + /// + /// The current load weight (0 = idle .. 255 = fully loaded). + /// + byte GetLoadWeight(); + + /// + /// Raised with the new load weight whenever it changes. + /// + event Action? LoadWeightChanged; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerDirectionPublisher.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerDirectionPublisher.cs new file mode 100644 index 0000000000..89b6caf26a --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerDirectionPublisher.cs @@ -0,0 +1,55 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Publishes the local Server's direction signals (health ServiceLevel and load weight) to the shared + /// store so peers can read them through an . + /// + public interface IPeerDirectionPublisher + { + /// + /// Publishes the local Server's current health ServiceLevel (eligibility signal). + /// + /// The health service level. + /// Cancellation token. + ValueTask PublishServiceLevelAsync(byte serviceLevel, CancellationToken cancellationToken = default); + + /// + /// Publishes the local Server's current load weight (tie-breaking signal). + /// + /// The load weight (0 = idle .. 255 = fully loaded). + /// Cancellation token. + ValueTask PublishLoadWeightAsync(byte loadWeight, CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerDirectionView.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerDirectionView.cs new file mode 100644 index 0000000000..a8e2be0f36 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerDirectionView.cs @@ -0,0 +1,50 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Reads the current, fresh direction inputs (health ServiceLevel + load weight) for every member of the + /// RedundantServerSet from the shared store. Stale records are aged out so a partitioned or lagging view + /// fails safe (the direction policy then returns the local Server). + /// + public interface IPeerDirectionView + { + /// + /// Returns the fresh per-peer direction records (including the local Server). A member is included only when + /// its health record is within the staleness window; a stale load record is reported as + /// = false. + /// + /// Cancellation token. + ValueTask> GetPeersAsync(CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerEndpointDirectory.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerEndpointDirectory.cs new file mode 100644 index 0000000000..03da465aa4 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerEndpointDirectory.cs @@ -0,0 +1,51 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Resolves the published EndpointDescriptions of a peer Server in the RedundantServerSet so the + /// local Server can return them from a redirected GetEndpoints response. + /// + public interface IPeerEndpointDirectory + { + /// + /// Returns the peer's published endpoints, or an empty set when none are available (unknown peer, missing, + /// or a record that failed integrity verification). + /// + /// The peer ServerUri / ApplicationUri. + /// Cancellation token. + ValueTask> GetEndpointsAsync( + string serverUri, + CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerEndpointPublisher.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerEndpointPublisher.cs new file mode 100644 index 0000000000..3bd76da8a3 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IPeerEndpointPublisher.cs @@ -0,0 +1,50 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Publishes the local Server's own EndpointDescriptions to the shared store so peers can return them when + /// they direct a Client to this Server. + /// + public interface IPeerEndpointPublisher + { + /// + /// Publishes the local Server's endpoints (integrity-protected) under its ServerUri. + /// + /// The endpoints to publish. + /// Cancellation token. + ValueTask PublishAsync( + ArrayOf endpoints, + CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IServerDirectionPolicy.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IServerDirectionPolicy.cs new file mode 100644 index 0000000000..74429ee967 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/IServerDirectionPolicy.cs @@ -0,0 +1,57 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Decides, for a GetEndpoints request handled by the local Server, which member of the + /// RedundantServerSet a Client should be directed to. + /// + public interface IServerDirectionPolicy + { + /// + /// Returns the ServerUri of the peer the Client should be directed to, or null when the local Server + /// should serve the request itself. The decision is: eligibility by health ServiceLevel tier first + /// (a peer in a strictly higher tier wins), then least load among the top tier (with the local Server + /// included), with random tie-breaking among equally-loaded members and fail-to-self on a stale/unknown view. + /// + /// The local ServerUri. + /// The local health ServiceLevel. + /// The local load weight. + /// Cancellation token. + ValueTask SelectTargetServerUriAsync( + string localServerUri, + byte localServiceLevel, + byte localLoadWeight, + CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionBuilderExtensions.cs new file mode 100644 index 0000000000..955b3170fd --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionBuilderExtensions.cs @@ -0,0 +1,117 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Fluent registration of the extension-beyond-§6.6 GetEndpoints load-direction feature that directs a + /// Client to the best member of a RedundantServerSet. + /// + public static class LoadDirectionBuilderExtensions + { + /// + /// Registers GetEndpoints load direction: peers publish their health ServiceLevel, load weight, + /// and endpoints to the shared store, and a request on the configured balancing discovery URL is answered with + /// the best peer's endpoints. Requires a shared store (register UseReplicatedAddressSpace / + /// UseRedundancyConsistency); pair with AddServerServiceLevel(...) for a real health signal and + /// register an for load-aware balancing. It complements — and never + /// replaces — the standard client-driven RedundantServerArray / ServiceLevel Failover. + /// + /// The server builder. + /// Configures the load-direction options (at minimum the balancing URL). + /// is null. + public static IOpcUaServerBuilder UseServerLoadDirection( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new LoadDirectionOptions(); + configure?.Invoke(options); + builder.Services.AddSingleton(options); + builder.Services.TryAddSingleton(_ => new ConstantLoadWeightProvider()); + + if (options.StrongEligibility) + { + builder.Services.AddSingleton( + new LoadDirectionStrongKeyspaceProvider(options)); + } + + builder.Services.AddSingleton(sp => new ServerLoadDirector( + ResolveServiceLevelProvider(sp), + sp.GetRequiredService(), + options, + sp.GetService>())); + builder.Services.AddSingleton( + sp => sp.GetRequiredService()); + + builder.Services.AddSingleton(sp => new LoadDirectionStartupTask( + sp.GetRequiredService(), + ResolveProtector(sp), + options, + sp.GetRequiredService(), + ResolveTimeProvider(sp))); + builder.Services.AddSingleton(sp => new PeerDirectionPublishStartupTask( + sp.GetRequiredService(), + ResolveProtector(sp), + options, + ResolveServiceLevelProvider(sp), + sp.GetRequiredService(), + ResolveTimeProvider(sp))); + + return builder; + } + + private static IServiceLevelProvider ResolveServiceLevelProvider(IServiceProvider serviceProvider) + { + return serviceProvider.GetService() ?? new ConstantServiceLevelProvider(); + } + + private static IRecordProtector ResolveProtector(IServiceProvider serviceProvider) + { + return serviceProvider.GetService() ?? NullRecordProtector.Instance; + } + + private static TimeProvider ResolveTimeProvider(IServiceProvider serviceProvider) + { + return serviceProvider.GetService() ?? TimeProvider.System; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionOptions.cs new file mode 100644 index 0000000000..88148a5f99 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionOptions.cs @@ -0,0 +1,103 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Options for the extension-beyond-§6.6 GetEndpoints load-direction feature that publishes and reads the + /// per-peer health ServiceLevel and load weight used to direct a Client to the best Server in a + /// RedundantServerSet. + /// + public sealed class LoadDirectionOptions + { + /// + /// The shared-store key prefix for the per-peer health ServiceLevel signal (eligibility). This keyspace + /// can be routed to the strongly-consistent store (Raft) for a deterministic redirect target. + /// + public string ServiceLevelKeyPrefix { get; set; } = "svc/"; + + /// + /// The shared-store key prefix for the per-peer load weight (tie-breaking only). This keyspace is always + /// eventually consistent and coalesced, because it is high-churn and a stale load tie-break is harmless. + /// + public string LoadKeyPrefix { get; set; } = "load/"; + + /// + /// The shared-store key prefix for the per-peer published EndpointDescriptions that the local Server + /// returns when it directs a Client to a peer. Low-churn (published at startup and on certificate rotation), + /// so it can be routed to the strongly-consistent store alongside the health signal. + /// + public string EndpointKeyPrefix { get; set; } = "endpoint/"; + + /// + /// How long a gossiped per-peer record is considered fresh. A peer whose health record is older than this is + /// excluded from direction (fail-safe); a load record older than this is treated as unknown for tie-breaking. + /// + public TimeSpan StalenessWindow { get; set; } = TimeSpan.FromSeconds(15); + + /// + /// The minimum interval between load-weight publishes. Coalesces high-churn load updates into at most one + /// write per interval so a per-tick shared-store write (and, under strong consistency, per-tick quorum) is + /// avoided. + /// + public TimeSpan LoadPublishInterval { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Sub-band size (in ServiceLevel units) applied within the Healthy range (200–255) when ranking + /// eligibility. The default 0 treats the entire Healthy range as a single eligibility tier, so equally + /// healthy peers are load-balanced by the separate load weight rather than redirected on minor health jitter. + /// A positive value sub-divides the Healthy range so a meaningfully healthier peer is preferred. + /// + public int HealthSubBandSize { get; set; } + + /// + /// Quantization band size (in load-weight units) applied when tie-breaking equally-eligible peers by load. + /// Peers whose load falls in the same band are treated as tied and one is chosen at random, damping the + /// herd/oscillation that a strict least-loaded choice would cause. + /// + public int LoadBandSize { get; set; } = 16; + + /// + /// The dedicated balancing discovery URL that opts a Client into load direction. A GetEndpoints request + /// whose endpointUrl matches this value may be answered with a peer's endpoints; requests to any other + /// (normal) discovery URL are unaffected. When empty, load direction never redirects (publish-only). + /// + public string BalancingEndpointUrl { get; set; } = string.Empty; + + /// + /// When true, the eligibility keyspaces (health ServiceLevel and the endpoint directory) are + /// routed to the linearizable (Raft) store in the eventual/hybrid consistency mode, giving a deterministic + /// redirect target; the high-churn load weight always stays eventual. Only takes effect when the shared store + /// is configured with UseRedundancyConsistency. Default false (all direction keyspaces eventual). + /// + public bool StrongEligibility { get; set; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionRandom.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionRandom.cs new file mode 100644 index 0000000000..304e1c37d8 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionRandom.cs @@ -0,0 +1,56 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Thread-safe random index selector used to break load ties among equally-eligible peers without a shared + /// counter (each replica chooses independently, spreading load across the tied set). + /// + internal static class LoadDirectionRandom + { + /// + /// Returns a random index in [0, exclusiveMax), or 0 when there is at most one choice. + /// + public static int NextIndex(int exclusiveMax) + { + // CA5394: the tie-break only spreads load fairly across equally-eligible peers; it is not a security + // decision, so a fast non-cryptographic PRNG is the correct choice. +#pragma warning disable CA5394 + return exclusiveMax <= 1 ? 0 : s_random.Value!.Next(exclusiveMax); +#pragma warning restore CA5394 + } + + private static readonly ThreadLocal s_random = new( + () => new Random(unchecked((Environment.TickCount * 397) ^ Environment.CurrentManagedThreadId))); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionStartupTask.cs new file mode 100644 index 0000000000..98e4da5683 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionStartupTask.cs @@ -0,0 +1,103 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Server startup task that activates the once the server is started: it builds + /// the context-dependent collaborators (peer view, direction policy, endpoint directory and publisher) with the + /// populated server message context and the local ServerUri, then calls . + /// + public sealed class LoadDirectionStartupTask : IServerStartupTask + { + /// + /// Creates the task. + /// + /// The shared store the direction signals and endpoints are gossiped through. + /// Protects record integrity. + /// The load-direction options. + /// The director to activate. + /// The time source for staleness checks. + public LoadDirectionStartupTask( + ISharedKeyValueStore store, + IRecordProtector protector, + LoadDirectionOptions options, + ServerLoadDirector director, + TimeProvider timeProvider) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_protector = protector ?? throw new ArgumentNullException(nameof(protector)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_director = director ?? throw new ArgumentNullException(nameof(director)); + m_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + /// + public ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + string[] serverUris = server.ServerUris.ToArray(); + string? localServerUri = serverUris.Length > 0 ? serverUris[0] : null; + if (string.IsNullOrEmpty(localServerUri)) + { + server.Telemetry.CreateLogger().LogWarning( + "Load direction disabled: the local ServerUri is unavailable."); + return default; + } + + IServiceMessageContext context = server.MessageContext; + var view = new SharedPeerDirectionView(m_store, context, m_protector, m_options, m_timeProvider); + var policy = new BandedServerDirectionPolicy(view, m_options, LoadDirectionRandom.NextIndex); + var directory = new SharedPeerEndpointDirectory(m_store, context, m_protector, m_options); + var endpointPublisher = new SharedPeerEndpointPublisher( + m_store, context, m_protector, m_options, localServerUri!); + + m_director.Configure(policy, directory, endpointPublisher, localServerUri!); + return default; + } + + private readonly ISharedKeyValueStore m_store; + private readonly IRecordProtector m_protector; + private readonly LoadDirectionOptions m_options; + private readonly ServerLoadDirector m_director; + private readonly TimeProvider m_timeProvider; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionStrongKeyspaceProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionStrongKeyspaceProvider.cs new file mode 100644 index 0000000000..582fe6bd2b --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/LoadDirectionStrongKeyspaceProvider.cs @@ -0,0 +1,54 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Routes the load-direction eligibility keyspaces (health ServiceLevel and the endpoint directory) + /// to the linearizable store when is set; the high-churn load + /// keyspace is deliberately left eventual. + /// + internal sealed class LoadDirectionStrongKeyspaceProvider : IStrongKeyspaceProvider + { + public LoadDirectionStrongKeyspaceProvider(LoadDirectionOptions options) + { + m_options = options ?? throw new ArgumentNullException(nameof(options)); + } + + /// + public ArrayOf GetStrongKeyPrefixes() + { + return [m_options.ServiceLevelKeyPrefix, m_options.EndpointKeyPrefix]; + } + + private readonly LoadDirectionOptions m_options; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionCodec.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionCodec.cs new file mode 100644 index 0000000000..05be1f5a52 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionCodec.cs @@ -0,0 +1,98 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Compact binary encoding for a single gossiped direction signal: a (ServerUri, byte value, UTC-ticks timestamp) + /// triple. The ServiceLevel and load-weight keyspaces both use this format; the byte carries the + /// ServiceLevel or the load weight respectively. + /// + internal static class PeerDirectionCodec + { + private const byte Version = 1; + + /// + /// Encodes a signal into a self-describing payload. + /// + public static ByteString Encode( + string serverUri, + byte value, + long timestampTicks, + IServiceMessageContext context) + { + using var encoder = new BinaryEncoder(context); + encoder.WriteByte(null, Version); + encoder.WriteString(null, serverUri); + encoder.WriteByte(null, value); + encoder.WriteInt64(null, timestampTicks); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : new ByteString(buffer); + } + + /// + /// Decodes a signal payload, returning false when it is malformed or of an unknown version. + /// + public static bool TryDecode( + ByteString payload, + IServiceMessageContext context, + out string serverUri, + out byte value, + out long timestampTicks) + { + serverUri = string.Empty; + value = 0; + timestampTicks = 0; + + if (payload.IsNull || payload.Length == 0) + { + return false; + } + + try + { + using var decoder = new BinaryDecoder(payload.ToArray(), context); + if (decoder.ReadByte(null) != Version) + { + return false; + } + serverUri = decoder.ReadString(null) ?? string.Empty; + value = decoder.ReadByte(null); + timestampTicks = decoder.ReadInt64(null); + return serverUri.Length != 0; + } + catch (ServiceResultException) + { + return false; + } + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionPublishStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionPublishStartupTask.cs new file mode 100644 index 0000000000..2efe4141f1 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionPublishStartupTask.cs @@ -0,0 +1,197 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Server startup task that publishes the local Server's direction signals to the shared store: the health + /// ServiceLevel immediately whenever it changes, and the load weight coalesced to at most one write per + /// . Peers read these through an + /// to direct Clients to the best Server. + /// + public sealed class PeerDirectionPublishStartupTask : IServerStartupTask, IDisposable + { + /// + /// Creates the task. + /// + /// The shared store the signals are gossiped through. + /// Protects record integrity. + /// The load-direction options. + /// The local health service-level source. + /// The local load-weight source. + /// The time source for record timestamps. + public PeerDirectionPublishStartupTask( + ISharedKeyValueStore store, + IRecordProtector protector, + LoadDirectionOptions options, + IServiceLevelProvider serviceLevelProvider, + ILoadWeightProvider loadWeightProvider, + TimeProvider timeProvider) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_protector = protector ?? throw new ArgumentNullException(nameof(protector)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_serviceLevelProvider = serviceLevelProvider + ?? throw new ArgumentNullException(nameof(serviceLevelProvider)); + m_loadWeightProvider = loadWeightProvider ?? throw new ArgumentNullException(nameof(loadWeightProvider)); + m_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + /// + public async ValueTask OnServerStartedAsync( + IServerInternal server, + CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + string? localServerUri = ResolveLocalServerUri(server); + m_logger = server.Telemetry.CreateLogger(); + if (string.IsNullOrEmpty(localServerUri)) + { + m_logger.LogWarning( + "Load direction disabled: the local ServerUri is unavailable, so peers cannot be directed to this Server."); + return; + } + + m_publisher = new SharedPeerDirectionPublisher( + m_store, server.MessageContext, m_protector, m_options, m_timeProvider, localServerUri!); + + // Publish the current values once, then react to changes. + await PublishServiceLevelSafeAsync( + m_serviceLevelProvider.GetServiceLevel(), cancellationToken).ConfigureAwait(false); + + byte initialLoad = m_loadWeightProvider.GetLoadWeight(); + Volatile.Write(ref m_latestLoad, initialLoad); + m_lastPublishedLoad = initialLoad; + await PublishLoadSafeAsync(initialLoad, cancellationToken).ConfigureAwait(false); + + m_serviceLevelProvider.ServiceLevelChanged += OnServiceLevelChanged; + m_loadWeightProvider.LoadWeightChanged += OnLoadWeightChanged; + + if (m_options.LoadPublishInterval > TimeSpan.Zero) + { + m_loadTimer = new Timer( + OnLoadTick, null, m_options.LoadPublishInterval, m_options.LoadPublishInterval); + } + } + + /// + public void Dispose() + { + m_serviceLevelProvider.ServiceLevelChanged -= OnServiceLevelChanged; + m_loadWeightProvider.LoadWeightChanged -= OnLoadWeightChanged; + m_loadTimer?.Dispose(); + m_loadTimer = null; + } + + private void OnServiceLevelChanged(byte serviceLevel) + { + _ = PublishServiceLevelSafeAsync(serviceLevel, CancellationToken.None); + } + + private void OnLoadWeightChanged(byte loadWeight) + { + Volatile.Write(ref m_latestLoad, loadWeight); + } + + private void OnLoadTick(object? state) + { + int latest = Volatile.Read(ref m_latestLoad); + if (latest == m_lastPublishedLoad) + { + return; + } + m_lastPublishedLoad = latest; + _ = PublishLoadSafeAsync((byte)latest, CancellationToken.None); + } + + private async Task PublishServiceLevelSafeAsync(byte serviceLevel, CancellationToken cancellationToken) + { + IPeerDirectionPublisher? publisher = m_publisher; + if (publisher == null) + { + return; + } + try + { + await publisher.PublishServiceLevelAsync(serviceLevel, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + m_logger?.LogDebug(ex, "Failed to publish the health ServiceLevel direction signal."); + } + } + + private async Task PublishLoadSafeAsync(byte loadWeight, CancellationToken cancellationToken) + { + IPeerDirectionPublisher? publisher = m_publisher; + if (publisher == null) + { + return; + } + try + { + await publisher.PublishLoadWeightAsync(loadWeight, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + m_logger?.LogDebug(ex, "Failed to publish the load-weight direction signal."); + } + } + + private static string? ResolveLocalServerUri(IServerInternal server) + { + string[] serverUris = server.ServerUris.ToArray(); + return serverUris.Length > 0 ? serverUris[0] : null; + } + + private readonly ISharedKeyValueStore m_store; + private readonly IRecordProtector m_protector; + private readonly LoadDirectionOptions m_options; + private readonly IServiceLevelProvider m_serviceLevelProvider; + private readonly ILoadWeightProvider m_loadWeightProvider; + private readonly TimeProvider m_timeProvider; + private ILogger? m_logger; + private IPeerDirectionPublisher? m_publisher; + private Timer? m_loadTimer; + private int m_latestLoad; + private int m_lastPublishedLoad; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionRecord.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionRecord.cs new file mode 100644 index 0000000000..5b48a2098b --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerDirectionRecord.cs @@ -0,0 +1,61 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// A fresh, per-Server view of the direction inputs for one member of a RedundantServerSet: its health + /// ServiceLevel (eligibility) and, when known, its load weight (tie-breaking). + /// + public sealed record PeerDirectionRecord + { + /// + /// The peer ServerUri / ApplicationUri that identifies the member. + /// + public string ServerUri { get; init; } = string.Empty; + + /// + /// The peer's health ServiceLevel (OPC 10000-4 Table 105). Direction eligibility is decided by this + /// value only. + /// + public byte ServiceLevel { get; init; } + + /// + /// The peer's load weight (0 = idle .. 255 = fully loaded), used only to break ties among peers already tied + /// at the highest health band. Valid only when is true. + /// + public byte LoadWeight { get; init; } + + /// + /// Whether a fresh load weight was available for this peer. When false the load weight is unknown + /// (stale or never published) and the peer is treated as least-preferred for tie-breaking. + /// + public bool LoadKnown { get; init; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerEndpointCodec.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerEndpointCodec.cs new file mode 100644 index 0000000000..1bdb544983 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/PeerEndpointCodec.cs @@ -0,0 +1,83 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Compact binary encoding for a peer's published EndpointDescription set (URL + certificate + security and + /// user-token policies) used to return a peer's endpoints from a redirected GetEndpoints response. + /// + internal static class PeerEndpointCodec + { + private const byte Version = 1; + + /// + /// Encodes an endpoint set into a self-describing payload. + /// + public static ByteString Encode(ArrayOf endpoints, IServiceMessageContext context) + { + using var encoder = new BinaryEncoder(context); + encoder.WriteByte(null, Version); + encoder.WriteEncodeableArray(null, endpoints); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : new ByteString(buffer); + } + + /// + /// Decodes an endpoint payload, returning false when it is malformed or of an unknown version. + /// + public static bool TryDecode( + ByteString payload, + IServiceMessageContext context, + out ArrayOf endpoints) + { + endpoints = []; + + if (payload.IsNull || payload.Length == 0) + { + return false; + } + + try + { + using var decoder = new BinaryDecoder(payload.ToArray(), context); + if (decoder.ReadByte(null) != Version) + { + return false; + } + endpoints = decoder.ReadEncodeableArray(null); + return true; + } + catch (ServiceResultException) + { + return false; + } + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ServerLoadDirector.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ServerLoadDirector.cs new file mode 100644 index 0000000000..ec7a45ae28 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/ServerLoadDirector.cs @@ -0,0 +1,220 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Globalization; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Directs a GetEndpoints request to the best member of a RedundantServerSet when it arrives on the + /// dedicated balancing discovery URL, and publishes the local Server's own endpoints (observed from normal + /// discovery requests) so peers can direct Clients to it. Inactive until runs at startup; + /// a stale/unknown view or an unresolved target fails safe to the local Server. + /// + public sealed class ServerLoadDirector : IGetEndpointsDirector + { + /// + /// Creates the director with the always-available collaborators. + /// + /// The local health service-level source. + /// The local load-weight source. + /// The load-direction options (balancing URL). + /// Optional logger for best-effort publish failures. + public ServerLoadDirector( + IServiceLevelProvider serviceLevelProvider, + ILoadWeightProvider loadWeightProvider, + LoadDirectionOptions options, + ILogger? logger = null) + { + m_serviceLevelProvider = serviceLevelProvider + ?? throw new ArgumentNullException(nameof(serviceLevelProvider)); + m_loadWeightProvider = loadWeightProvider ?? throw new ArgumentNullException(nameof(loadWeightProvider)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_logger = logger; + } + + /// + /// Activates the director with the context-dependent collaborators built at server startup. + /// + /// The direction policy. + /// Resolves a peer's published endpoints. + /// Publishes the local Server's endpoints for peers. + /// The local ServerUri. + public void Configure( + IServerDirectionPolicy policy, + IPeerEndpointDirectory endpointDirectory, + IPeerEndpointPublisher endpointPublisher, + string localServerUri) + { + m_endpointDirectory = endpointDirectory ?? throw new ArgumentNullException(nameof(endpointDirectory)); + m_endpointPublisher = endpointPublisher ?? throw new ArgumentNullException(nameof(endpointPublisher)); + if (string.IsNullOrEmpty(localServerUri)) + { + throw new ArgumentException("The local ServerUri must be provided.", nameof(localServerUri)); + } + m_localServerUri = localServerUri; + m_policy = policy ?? throw new ArgumentNullException(nameof(policy)); + } + + /// + public async ValueTask<(bool Redirect, ArrayOf Endpoints)> TryGetDirectedEndpointsAsync( + string? endpointUrl, + ArrayOf localEndpoints, + CancellationToken cancellationToken = default) + { + IServerDirectionPolicy? policy = m_policy; + if (policy == null) + { + return (false, default); + } + + if (!IsBalancingEndpoint(endpointUrl)) + { + // Normal discovery request: publish our own endpoints for peers, then serve locally. + await MaybePublishLocalEndpointsAsync(localEndpoints, cancellationToken).ConfigureAwait(false); + return (false, default); + } + + string? localServerUri = m_localServerUri; + if (string.IsNullOrEmpty(localServerUri)) + { + return (false, default); + } + + string? target; + try + { + target = await policy.SelectTargetServerUriAsync( + localServerUri!, + m_serviceLevelProvider.GetServiceLevel(), + m_loadWeightProvider.GetLoadWeight(), + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + m_logger?.LogDebug(ex, "Load direction policy failed; serving local endpoints."); + return (false, default); + } + + if (target == null) + { + return (false, default); + } + + ArrayOf peerEndpoints; + try + { + peerEndpoints = await m_endpointDirectory! + .GetEndpointsAsync(target, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + m_logger?.LogDebug(ex, "Failed to resolve peer endpoints; serving local endpoints."); + return (false, default); + } + + // Fail safe: without the target's endpoints we cannot direct the Client, so serve locally. + return peerEndpoints.Count == 0 ? (false, default) : (true, peerEndpoints); + } + + private async ValueTask MaybePublishLocalEndpointsAsync( + ArrayOf localEndpoints, + CancellationToken cancellationToken) + { + IPeerEndpointPublisher? publisher = m_endpointPublisher; + if (publisher == null || localEndpoints.Count == 0) + { + return; + } + + string signature = ComputeSignature(localEndpoints); + if (string.Equals(Volatile.Read(ref m_lastPublishedSignature), signature, StringComparison.Ordinal)) + { + return; + } + Volatile.Write(ref m_lastPublishedSignature, signature); + + try + { + await publisher.PublishAsync(localEndpoints, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + m_logger?.LogDebug(ex, "Failed to publish local endpoints for load direction."); + } + } + + private bool IsBalancingEndpoint(string? endpointUrl) + { + string balancing = m_options.BalancingEndpointUrl; + if (string.IsNullOrEmpty(balancing) || string.IsNullOrEmpty(endpointUrl)) + { + return false; + } + return string.Equals(Normalize(endpointUrl!), Normalize(balancing), StringComparison.Ordinal); + } + + private static string Normalize(string url) + { + return url.Trim().TrimEnd('/').ToLowerInvariant(); + } + + private static string ComputeSignature(ArrayOf endpoints) + { + var builder = new StringBuilder(); + for (int ii = 0; ii < endpoints.Count; ii++) + { + EndpointDescription endpoint = endpoints[ii]; + builder.Append(endpoint.EndpointUrl) + .Append('|') + .Append(endpoint.SecurityPolicyUri) + .Append('|') + .Append(((int)endpoint.SecurityMode).ToString(CultureInfo.InvariantCulture)) + .Append(';'); + } + return builder.ToString(); + } + + private readonly IServiceLevelProvider m_serviceLevelProvider; + private readonly ILoadWeightProvider m_loadWeightProvider; + private readonly LoadDirectionOptions m_options; + private readonly ILogger? m_logger; + private IServerDirectionPolicy? m_policy; + private IPeerEndpointDirectory? m_endpointDirectory; + private IPeerEndpointPublisher? m_endpointPublisher; + private string? m_localServerUri; + private string? m_lastPublishedSignature; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerDirectionPublisher.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerDirectionPublisher.cs new file mode 100644 index 0000000000..cbbe3ec8c0 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerDirectionPublisher.cs @@ -0,0 +1,100 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Publishes the local Server's direction signals to an as integrity-protected, + /// timestamped records keyed by the local ServerUri. + /// + public sealed class SharedPeerDirectionPublisher : IPeerDirectionPublisher + { + /// + /// Creates the publisher. + /// + /// The shared store the signals are gossiped through. + /// The message context used to encode records. + /// Protects record integrity. + /// The load-direction options (key prefixes). + /// The time source for record timestamps. + /// The local ServerUri used as the record key. + /// is null or empty. + public SharedPeerDirectionPublisher( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector protector, + LoadDirectionOptions options, + TimeProvider timeProvider, + string localServerUri) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? throw new ArgumentNullException(nameof(protector)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + if (string.IsNullOrEmpty(localServerUri)) + { + throw new ArgumentException("The local ServerUri must be provided.", nameof(localServerUri)); + } + m_localServerUri = localServerUri; + } + + /// + public ValueTask PublishServiceLevelAsync(byte serviceLevel, CancellationToken cancellationToken = default) + { + return PublishAsync(m_options.ServiceLevelKeyPrefix, serviceLevel, cancellationToken); + } + + /// + public ValueTask PublishLoadWeightAsync(byte loadWeight, CancellationToken cancellationToken = default) + { + return PublishAsync(m_options.LoadKeyPrefix, loadWeight, cancellationToken); + } + + private ValueTask PublishAsync(string keyPrefix, byte value, CancellationToken cancellationToken) + { + long ticks = m_timeProvider.GetUtcNow().UtcDateTime.Ticks; + ByteString payload = m_protector.Protect( + PeerDirectionCodec.Encode(m_localServerUri, value, ticks, m_context)); + return m_store.SetAsync(keyPrefix + m_localServerUri, payload, cancellationToken); + } + + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + private readonly LoadDirectionOptions m_options; + private readonly TimeProvider m_timeProvider; + private readonly string m_localServerUri; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerDirectionView.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerDirectionView.cs new file mode 100644 index 0000000000..b51a4d7c61 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerDirectionView.cs @@ -0,0 +1,139 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Reads the per-peer health ServiceLevel and load-weight signals from an + /// , verifying record integrity (fail-closed) and aging out stale entries. + /// + public sealed class SharedPeerDirectionView : IPeerDirectionView + { + /// + /// Creates the view. + /// + /// The shared store the signals are gossiped through. + /// The message context used to decode records. + /// Verifies record integrity; forged/tampered records are dropped. + /// The load-direction options (key prefixes, staleness window). + /// The time source for staleness checks. + public SharedPeerDirectionView( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector protector, + LoadDirectionOptions options, + TimeProvider timeProvider) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? throw new ArgumentNullException(nameof(protector)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + /// + public async ValueTask> GetPeersAsync( + CancellationToken cancellationToken = default) + { + long nowTicks = m_timeProvider.GetUtcNow().UtcDateTime.Ticks; + long stalenessTicks = m_options.StalenessWindow.Ticks; + + Dictionary health = await ReadSignalsAsync( + m_options.ServiceLevelKeyPrefix, cancellationToken).ConfigureAwait(false); + Dictionary load = await ReadSignalsAsync( + m_options.LoadKeyPrefix, cancellationToken).ConfigureAwait(false); + + var records = new List(health.Count); + foreach (KeyValuePair entry in health) + { + if (IsStale(nowTicks, entry.Value.Ticks, stalenessTicks)) + { + continue; + } + + bool loadKnown = load.TryGetValue(entry.Key, out (byte Value, long Ticks) loadEntry) && + !IsStale(nowTicks, loadEntry.Ticks, stalenessTicks); + + records.Add(new PeerDirectionRecord + { + ServerUri = entry.Key, + ServiceLevel = entry.Value.Value, + LoadWeight = loadKnown ? loadEntry.Value : (byte)0, + LoadKnown = loadKnown + }); + } + + return new ArrayOf(records.ToArray()); + } + + private async ValueTask> ReadSignalsAsync( + string keyPrefix, + CancellationToken cancellationToken) + { + var result = new Dictionary(StringComparer.Ordinal); + await foreach (KeyValuePair entry in m_store + .ScanAsync(keyPrefix, cancellationToken) + .ConfigureAwait(false)) + { + if (!m_protector.TryUnprotect(entry.Value, out ByteString payload) || + !PeerDirectionCodec.TryDecode( + payload, m_context, out string serverUri, out byte value, out long ticks)) + { + continue; + } + + // Keep the newest record when a peer appears more than once. + if (!result.TryGetValue(serverUri, out (byte Value, long Ticks) existing) || + ticks > existing.Ticks) + { + result[serverUri] = (value, ticks); + } + } + + return result; + } + + private static bool IsStale(long nowTicks, long recordTicks, long stalenessTicks) + { + return nowTicks - recordTicks > stalenessTicks; + } + + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + private readonly LoadDirectionOptions m_options; + private readonly TimeProvider m_timeProvider; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerEndpointDirectory.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerEndpointDirectory.cs new file mode 100644 index 0000000000..0488e9ba08 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerEndpointDirectory.cs @@ -0,0 +1,91 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Reads a peer's published EndpointDescriptions from an , verifying + /// record integrity (fail-closed). + /// + public sealed class SharedPeerEndpointDirectory : IPeerEndpointDirectory + { + /// + /// Creates the directory. + /// + /// The shared store the endpoints are gossiped through. + /// The message context used to decode records. + /// Verifies record integrity; forged/tampered records are dropped. + /// The load-direction options (endpoint key prefix). + public SharedPeerEndpointDirectory( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector protector, + LoadDirectionOptions options) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? throw new ArgumentNullException(nameof(protector)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + } + + /// + public async ValueTask> GetEndpointsAsync( + string serverUri, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrEmpty(serverUri)) + { + return []; + } + + (bool found, ByteString value) = await m_store + .TryGetAsync(m_options.EndpointKeyPrefix + serverUri, cancellationToken) + .ConfigureAwait(false); + + if (found && + m_protector.TryUnprotect(value, out ByteString payload) && + PeerEndpointCodec.TryDecode(payload, m_context, out ArrayOf endpoints)) + { + return endpoints; + } + + return []; + } + + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + private readonly LoadDirectionOptions m_options; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerEndpointPublisher.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerEndpointPublisher.cs new file mode 100644 index 0000000000..8a416981ee --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/LoadDirection/SharedPeerEndpointPublisher.cs @@ -0,0 +1,85 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Publishes the local Server's EndpointDescriptions to an as an + /// integrity-protected record keyed by the local ServerUri. + /// + public sealed class SharedPeerEndpointPublisher : IPeerEndpointPublisher + { + /// + /// Creates the publisher. + /// + /// The shared store the endpoints are gossiped through. + /// The message context used to encode records. + /// Protects record integrity. + /// The load-direction options (endpoint key prefix). + /// The local ServerUri used as the record key. + /// is null or empty. + public SharedPeerEndpointPublisher( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector protector, + LoadDirectionOptions options, + string localServerUri) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? throw new ArgumentNullException(nameof(protector)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + if (string.IsNullOrEmpty(localServerUri)) + { + throw new ArgumentException("The local ServerUri must be provided.", nameof(localServerUri)); + } + m_localServerUri = localServerUri; + } + + /// + public ValueTask PublishAsync( + ArrayOf endpoints, + CancellationToken cancellationToken = default) + { + ByteString payload = m_protector.Protect(PeerEndpointCodec.Encode(endpoints, m_context)); + return m_store.SetAsync(m_options.EndpointKeyPrefix + m_localServerUri, payload, cancellationToken); + } + + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + private readonly LoadDirectionOptions m_options; + private readonly string m_localServerUri; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RedundantPeer.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RedundantPeer.cs new file mode 100644 index 0000000000..9d3bb1059a --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RedundantPeer.cs @@ -0,0 +1,72 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Describes a peer Server in a non-transparent RedundantServerSet. + /// + public sealed class RedundantPeer + { + /// + /// Creates an empty peer description. + /// + public RedundantPeer() + { + } + + /// + /// Creates a peer description. + /// + /// The peer ServerUri / ApplicationUri. + /// The peer discovery URLs. + public RedundantPeer(string applicationUri, ArrayOf discoveryUrls) + { + ApplicationUri = applicationUri; + DiscoveryUrls = discoveryUrls; + } + + /// + /// Gets or sets the peer ServerUri / ApplicationUri. + /// + public string ApplicationUri { get; set; } = string.Empty; + + /// + /// Gets or sets the peer Server application name. + /// + public LocalizedText ApplicationName { get; set; } = LocalizedText.Null; + + /// + /// Gets or sets the peer discovery URLs. + /// + public ArrayOf DiscoveryUrls { get; set; } = []; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RequestServerStateChangeOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RequestServerStateChangeOptions.cs new file mode 100644 index 0000000000..9d13ab3d54 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RequestServerStateChangeOptions.cs @@ -0,0 +1,60 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Options for wiring the standard OPC 10000-4 §6.6.5 Server.RequestServerStateChange method. + /// + public sealed class RequestServerStateChangeOptions + { + /// + /// Gets or sets an optional administrator access validator. + /// + /// + /// When unset, the startup task uses + /// . + /// The current implementation publishes the requested Maintenance or + /// NoData ServiceLevel so Clients back off. It does not install + /// a transport-level hook to reject newly created sessions. + /// + public Action? AdminAccessValidator { get; set; } + + /// + /// Gets or sets an optional mapping from requested server state to + /// ServiceLevel. + /// + public Func? ServiceLevelSelector { get; set; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RequestServerStateChangeStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RequestServerStateChangeStartupTask.cs new file mode 100644 index 0000000000..04f56cbac1 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/RequestServerStateChangeStartupTask.cs @@ -0,0 +1,213 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Wires the standard Server.RequestServerStateChange method for Maintenance-driven Failover + /// (OPC 10000-4 §6.6.5). + /// + public sealed class RequestServerStateChangeStartupTask : IServerStartupTask + { + /// + /// Creates the task. + /// + /// The method wiring options. + /// Optional service-level controller. + public RequestServerStateChangeStartupTask( + RequestServerStateChangeOptions options, + IServiceLevelController? serviceLevelController = null) + { + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_serviceLevelController = serviceLevelController; + } + + /// + /// Creates the task with default options. + /// + /// Optional service-level controller. + public RequestServerStateChangeStartupTask(IServiceLevelController? serviceLevelController = null) + : this(new RequestServerStateChangeOptions(), serviceLevelController) + { + } + + /// + public ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + ServerObjectState? serverObject = server.ServerObject; + if (serverObject == null) + { + return default; + } + + m_server = server; + RequestServerStateChangeMethodState? requestServerStateChange = + server.DiagnosticsNodeManager?.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange) ?? + serverObject.RequestServerStateChange; + if (requestServerStateChange != null) + { + requestServerStateChange.OnCall = OnRequestServerStateChange; + } + + return default; + } + + private ServiceResult OnRequestServerStateChange( + ISystemContext context, + MethodState method, + NodeId objectId, + ServerState state, + DateTimeUtc estimatedReturnTime, + uint secondsTillShutdown, + LocalizedText reason, + bool restart) + { + try + { + ValidateAdminAccess(context); + + IServerInternal? server = m_server; + ServerObjectState? serverObject = server?.ServerObject; + if (server == null || serverObject == null) + { + return new ServiceResult(StatusCodes.BadServerHalted); + } + + byte serviceLevel = SelectServiceLevel(state); + ApplyServiceLevel(serverObject, server.DefaultSystemContext, serviceLevel); + ApplyServerStatus(serverObject, server.DefaultSystemContext, state, estimatedReturnTime, + secondsTillShutdown, reason); + return ServiceResult.Good; + } + catch (ServiceResultException sre) + { + return sre.Result; + } + } + + private void ValidateAdminAccess(ISystemContext context) + { + if (m_options.AdminAccessValidator != null) + { + m_options.AdminAccessValidator(context); + return; + } + + IConfigurationNodeManager? configurationNodeManager = m_server?.ConfigurationNodeManager; + if (configurationNodeManager == null) + { + throw new ServiceResultException( + StatusCodes.BadUserAccessDenied, + "A configuration node manager is required to validate administrator access."); + } + + configurationNodeManager.HasApplicationSecureAdminAccess(context); + } + + private byte SelectServiceLevel(ServerState state) + { + if (m_options.ServiceLevelSelector != null) + { + return m_options.ServiceLevelSelector(state); + } + + return state switch + { + ServerState.Running => ServiceLevels.Maximum, + ServerState.Shutdown => ServiceLevels.Maintenance, + ServerState.Suspended => ServiceLevels.Maintenance, + _ => ServiceLevels.NoData + }; + } + + private void ApplyServiceLevel(ServerObjectState serverObject, ISystemContext context, byte serviceLevel) + { + if (m_serviceLevelController != null) + { + m_serviceLevelController.SetServiceLevel(serviceLevel); + return; + } + + if (serverObject.ServiceLevel != null) + { + serverObject.ServiceLevel.Value = serviceLevel; + serverObject.ServiceLevel.ClearChangeMasks(context, false); + } + } + + private static void ApplyServerStatus( + ServerObjectState serverObject, + ISystemContext context, + ServerState state, + DateTimeUtc estimatedReturnTime, + uint secondsTillShutdown, + LocalizedText reason) + { + if (serverObject.EstimatedReturnTime != null) + { + serverObject.EstimatedReturnTime.Value = estimatedReturnTime; + serverObject.EstimatedReturnTime.ClearChangeMasks(context, false); + } + + if (serverObject.ServerStatus?.State != null) + { + serverObject.ServerStatus.State.Value = state; + serverObject.ServerStatus.State.ClearChangeMasks(context, false); + } + if (serverObject.ServerStatus?.SecondsTillShutdown != null) + { + serverObject.ServerStatus.SecondsTillShutdown.Value = secondsTillShutdown; + serverObject.ServerStatus.SecondsTillShutdown.ClearChangeMasks(context, false); + } + if (serverObject.ServerStatus?.ShutdownReason != null) + { + serverObject.ServerStatus.ShutdownReason.Value = reason; + serverObject.ServerStatus.ShutdownReason.ClearChangeMasks(context, false); + } + } + + private readonly RequestServerStateChangeOptions m_options; + private readonly IServiceLevelController? m_serviceLevelController; + private IServerInternal? m_server; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyBuilderExtensions.cs new file mode 100644 index 0000000000..a8453a8033 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyBuilderExtensions.cs @@ -0,0 +1,137 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Opc.Ua.Configuration; +using Opc.Ua.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Fluent registration of OPC 10000-4 §6.6 server redundancy metadata publishing on the + /// . + /// + public static class ServerRedundancyBuilderExtensions + { + /// + /// Populates the live Server.ServerRedundancy model from the + /// supplied configuration after the hosted server starts. Non-transparent redundancy depends on + /// Server.ServiceLevel changes for client Failover decisions; pair this with + /// AddServerServiceLevel(new LeaderServiceLevelProvider(...)) or another + /// registered in dependency injection. + /// + /// The server builder. + /// Optional server redundancy configuration. + public static IOpcUaServerBuilder AddServerRedundancy( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new ServerRedundancyOptions(); + configure?.Invoke(options); + AddDiscoveryCapabilityConfiguration(builder, options); + builder.Services.AddSingleton(options); + builder.Services.AddSingleton( + new ConfiguredRedundantServerSetProvider(options)); + builder.Services.AddSingleton(sp => + new ServerRedundancyStartupTask( + options, + !sp.GetServices().Any())); + return builder; + } + + /// + /// Wires the standard Server.RequestServerStateChange method for OPC 10000-4 §6.6.5 + /// administrator-driven manual Failover to Maintenance or NoData. + /// + /// The server builder. + /// Optional method wiring configuration. + public static IOpcUaServerBuilder AddRequestServerStateChange( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new RequestServerStateChangeOptions(); + configure?.Invoke(options); + builder.Services.AddSingleton(sp => + new RequestServerStateChangeStartupTask( + options, + sp.GetService() as IServiceLevelController)); + return builder; + } + + /// + /// Wires the standard Server.RequestServerStateChange method for OPC 10000-4 §6.6.5 + /// administrator-driven manual Failover to Maintenance or NoData. + /// + /// The server builder. + /// Optional method wiring configuration. + /// + /// Use . + /// + [Obsolete("Use AddRequestServerStateChange.")] + public static IOpcUaServerBuilder AddManualFailover( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + return AddRequestServerStateChange(builder, configure); + } + + private static void AddDiscoveryCapabilityConfiguration( + IOpcUaServerBuilder builder, + ServerRedundancyOptions redundancyOptions) + { + builder.Services.AddOptions().Configure(serverOptions => + { + Action? previous = + serverOptions.ConfigureBuilder; + serverOptions.ConfigureBuilder = configurationBuilder => + { + previous?.Invoke(configurationBuilder); + if (redundancyOptions.IsNonTransparentMode && + redundancyOptions.AdvertiseNtrsCapability) + { + configurationBuilder.AddServerCapabilities("NTRS"); + } + }; + }); + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyOptions.cs new file mode 100644 index 0000000000..94ab1ccf85 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyOptions.cs @@ -0,0 +1,118 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Options used to publish the local Server's OPC 10000-4 §6.6.2 redundancy metadata in the standard + /// Server.ServerRedundancy nodes. + /// + public sealed class ServerRedundancyOptions + { + /// + /// Gets or sets the OPC UA redundancy mode reported by + /// Server.ServerRedundancy.RedundancySupport. + /// + public RedundancySupport Mode { get; set; } = RedundancySupport.None; + + /// + /// Gets URI-only fallback peer ServerUris for legacy configuration that cannot provide discovery URLs. + /// Prefer or as the canonical redundancy + /// peer input; those entries drive both FindServers and the flat redundancy URI variables. + /// + public IList PeerServerUris { get; } = new List(); + + /// + /// Gets the canonical rich peer descriptions used by FindServers for a non-transparent + /// RedundantServerSet (OPC 10000-4 §6.6.2.4.5.1) and by the flat redundancy URI variables. + /// + public IList RedundantPeers { get; } = new List(); + + /// + /// Gets or sets this Server's identifier within a Transparent RedundantServerSet. + /// + public string CurrentServerId { get; set; } = Environment.MachineName; + + /// + /// Gets or sets the ServiceLevel published for each configured peer Server in + /// Server.ServerRedundancy.RedundantServerArray. + /// + public byte PeerServiceLevel { get; set; } = ServiceLevels.Maximum; + + /// + /// Gets or sets a value indicating whether non-transparent modes add + /// the NTRS discovery capability. + /// + public bool AdvertiseNtrsCapability { get; set; } = true; + + /// + /// Gets a value indicating whether the configured mode is non-transparent redundancy. + /// + public bool IsNonTransparentMode => + Mode is RedundancySupport.Cold or + RedundancySupport.Warm or + RedundancySupport.Hot or + RedundancySupport.HotAndMirrored; + + /// + /// Adds a canonical redundant peer description. + /// + /// The peer application URI. + /// The peer discovery URLs. + /// The added peer. + public RedundantPeer AddRedundantPeer(string applicationUri, ArrayOf discoveryUrls) + { + var peer = new RedundantPeer(applicationUri, discoveryUrls); + RedundantPeers.Add(peer); + return peer; + } + + /// + /// Gets the peer application URIs used for redundancy variables, derived from the canonical + /// list plus the URI-only fallback. + /// + /// The configured peer application URIs. + public ArrayOf GetPeerApplicationUris() + { + return new ArrayOf(RedundantPeers + .Select(peer => peer.ApplicationUri) + .Concat(PeerServerUris) + .Where(uri => !string.IsNullOrEmpty(uri)) + .Distinct(StringComparer.Ordinal) + .ToArray()); + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyStartupTask.cs new file mode 100644 index 0000000000..8de7e2eb9f --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServerRedundancyStartupTask.cs @@ -0,0 +1,210 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Server startup task that populates the live + /// Server.ServerRedundancy nodes from + /// . + /// + public sealed class ServerRedundancyStartupTask : IServerStartupTask + { + /// + /// Creates the task. + /// + /// The server redundancy options. + /// + /// Whether to warn when non-transparent redundancy has no registered service-level provider. + /// + public ServerRedundancyStartupTask( + ServerRedundancyOptions options, + bool warnIfServiceLevelProviderMissing = false) + { + m_options = options ?? throw new ArgumentNullException(nameof(options)); + m_warnIfServiceLevelProviderMissing = warnIfServiceLevelProviderMissing; + } + + /// + public ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + WarnIfServiceLevelProviderMissing(server); + + ServerObjectState? serverObject = server.ServerObject; + ServerRedundancyState? redundancy = serverObject?.ServerRedundancy; + if (redundancy == null) + { + return default; + } + + ISystemContext context = server.DefaultSystemContext; + if (redundancy.RedundancySupport != null) + { + redundancy.RedundancySupport.Value = m_options.Mode; + redundancy.RedundancySupport.ClearChangeMasks(context, false); + } + ApplyServerRedundancyTypeDefinition(redundancy); + redundancy.ClearChangeMasks(context, false); + + if (m_options.Mode != RedundancySupport.None) + { + ApplyRedundantServerArray(server, redundancy, context); + if (m_options.Mode == RedundancySupport.Transparent) + { + ApplyCurrentServerId(server, redundancy, context); + } + else + { + ApplyServerUriArray(server, redundancy, context); + } + } + + return default; + } + + private void WarnIfServiceLevelProviderMissing(IServerInternal server) + { + if (!m_warnIfServiceLevelProviderMissing || !m_options.IsNonTransparentMode) + { + return; + } + + ILogger logger = server.Telemetry.CreateLogger(); + logger.LogWarning( + "Non-transparent server redundancy mode {RedundancyMode} is configured without a registered " + + "IServiceLevelProvider. Register AddServerServiceLevel(...) or an IServiceLevelProvider so " + + "Server.ServiceLevel reflects failover health.", + m_options.Mode); + } + + private void ApplyRedundantServerArray( + IServerInternal server, + ServerRedundancyState redundancy, + ISystemContext context) + { + PropertyState>? redundantServerArray = + server.DiagnosticsNodeManager?.FindPredefinedNode>>( + VariableIds.Server_ServerRedundancy_RedundantServerArray) ?? + redundancy.RedundantServerArray; + if (redundantServerArray == null) + { + return; + } + + ArrayOf peerApplicationUris = m_options.GetPeerApplicationUris(); + var servers = new List(peerApplicationUris.Count); + foreach (string peerServerUri in peerApplicationUris) + { + servers.Add(new RedundantServerDataType + { + ServerId = peerServerUri, + ServiceLevel = m_options.PeerServiceLevel, + ServerState = ServerState.Running + }); + } + + redundantServerArray.Value = new ArrayOf(servers.ToArray()); + redundantServerArray.ClearChangeMasks(context, false); + } + + private void ApplyCurrentServerId( + IServerInternal server, + ServerRedundancyState redundancy, + ISystemContext context) + { + PropertyState? currentServerId = + server.DiagnosticsNodeManager?.FindPredefinedNode>( + VariableIds.Server_ServerRedundancy_CurrentServerId) ?? + FindProperty(redundancy, context, BrowseNames.CurrentServerId); + if (currentServerId != null) + { + currentServerId.Value = m_options.CurrentServerId; + currentServerId.ClearChangeMasks(context, false); + } + } + + private void ApplyServerUriArray( + IServerInternal server, + ServerRedundancyState redundancy, + ISystemContext context) + { + PropertyState>? serverUriArray = + server.DiagnosticsNodeManager?.FindPredefinedNode>>( + VariableIds.Server_ServerRedundancy_ServerUriArray) ?? + FindProperty>(redundancy, context, BrowseNames.ServerUriArray); + if (serverUriArray != null) + { + serverUriArray.Value = m_options.GetPeerApplicationUris(); + serverUriArray.ClearChangeMasks(context, false); + } + } + + private static PropertyState? FindProperty( + ServerRedundancyState redundancy, + ISystemContext context, + string browseName) + { + NodeState? child = redundancy.FindChild(context, new QualifiedName(browseName, 0)); + return child as PropertyState; + } + + private static void ApplyServerRedundancyTypeDefinition(ServerRedundancyState redundancy) + { + redundancy.TypeDefinitionId = redundancy.RedundancySupport?.Value switch + { + RedundancySupport.Transparent => TransparentRedundancyTypeId, + RedundancySupport.Cold or + RedundancySupport.Warm or + RedundancySupport.Hot or + RedundancySupport.HotAndMirrored => NonTransparentRedundancyTypeId, + _ => ServerRedundancyTypeId + }; + } + + private readonly ServerRedundancyOptions m_options; + private readonly bool m_warnIfServiceLevelProviderMissing; + private static readonly NodeId ServerRedundancyTypeId = new(2034); + private static readonly NodeId TransparentRedundancyTypeId = new(2036); + private static readonly NodeId NonTransparentRedundancyTypeId = new(2039); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServiceLevelStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServiceLevelStartupTask.cs new file mode 100644 index 0000000000..d9ecd83769 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Redundancy/ServiceLevelStartupTask.cs @@ -0,0 +1,90 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Server startup task that drives the live Server.ServiceLevel + /// node from an : it sets the initial + /// value and updates the node (firing notifications) whenever the + /// provider's level changes. With the default + /// this reports a fixed 255, + /// preserving single-instance behavior. + /// + public sealed class ServiceLevelStartupTask : IServerStartupTask + { + /// + /// Creates the task. + /// + /// The service-level source. + public ServiceLevelStartupTask(IServiceLevelProvider serviceLevelProvider) + { + m_serviceLevelProvider = serviceLevelProvider + ?? throw new ArgumentNullException(nameof(serviceLevelProvider)); + } + + /// + public ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + ServerObjectState? serverObject = server.ServerObject; + if (serverObject?.ServiceLevel == null) + { + return default; + } + + ISystemContext context = server.DefaultSystemContext; + ApplyLevel(serverObject, context, m_serviceLevelProvider.GetServiceLevel()); + m_serviceLevelProvider.ServiceLevelChanged += level => ApplyLevel(serverObject, context, level); + return default; + } + + private static void ApplyLevel(ServerObjectState serverObject, ISystemContext context, byte level) + { + if (serverObject.ServiceLevel == null) + { + return; + } + serverObject.ServiceLevel.Value = level; + serverObject.ServiceLevel.ClearChangeMasks(context, false); + } + + private readonly IServiceLevelProvider m_serviceLevelProvider; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/RedundancyConsistencyBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/RedundancyConsistencyBuilderExtensions.cs new file mode 100644 index 0000000000..5d75d7ee4d --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/RedundancyConsistencyBuilderExtensions.cs @@ -0,0 +1,165 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: fluent selection of the shared-store consistency model (strong Raft vs. + /// eventual CRDT complemented by Raft) on the . + /// + public static class RedundancyConsistencyBuilderExtensions + { + /// + /// Registers the consistency-mode-appropriate , the shared + /// replica, and (by default) a native . + /// + /// + /// + /// In mode the shared store is a linearizable + /// . In mode (the + /// default) it is a that routes bulk keys to the CRDT bulk store and + /// the strong-prefix keyspaces (single-use nonces, lease, election) to Raft. + /// + /// + /// Call this before / + /// : those use + /// TryAddSingleton for the same services, so the consistency registration wins and the distributed + /// features compose over the chosen store and election. + /// + /// + /// The server builder. + /// Optional consistency options. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseRedundancyConsistency( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new RedundancyConsistencyOptions(); + configure?.Invoke(options); + + builder.Services.TryAddSingleton(sp => + options.RaftConsensusFactory?.Invoke(sp) ?? RaftCsConsensus.CreateSingleNode(options.NodeId)); + + builder.Services.TryAddSingleton(sp => + CreateStore(sp, options)); + + if (options.UseRaftLeaderElection) + { + builder.Services.TryAddSingleton(sp => + new RaftLeaderElection( + sp.GetRequiredService(), + sp.GetService()?.CreateLogger())); + } + + return builder; + } + + /// + /// Registers the shared store and election for the given consistency using defaults. + /// + /// The server builder. + /// The consistency model to use. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseRedundancyConsistency( + this IOpcUaServerBuilder builder, + RedundancyConsistencyMode mode) + { + return builder.UseRedundancyConsistency(options => options.Mode = mode); + } + + private static ISharedKeyValueStore CreateStore( + IServiceProvider sp, + RedundancyConsistencyOptions options) + { + IRaftConsensus consensus = sp.GetRequiredService(); + + // CA2000: this is a DI factory; ownership of every store created + // here transfers to the returned ISharedKeyValueStore (the Hybrid + // owns the Raft + bulk stores via ownsStores: true) and ultimately + // to the container, which disposes the singleton. +#pragma warning disable CA2000 + // The consensus replica is a container-owned singleton shared with + // the leader election, so the store must not dispose it. + var raftStore = new RaftSharedKeyValueStore(consensus, ownsConsensus: false); + if (options.Mode == RedundancyConsistencyMode.Strong) + { + return raftStore; + } + + ISharedKeyValueStore bulk = options.BulkStoreFactory?.Invoke(sp) + ?? new InMemorySharedKeyValueStore(); + return new HybridSharedKeyValueStore( + bulk, + raftStore, + AggregateStrongKeyPrefixes(sp, options), + ownsStores: true); +#pragma warning restore CA2000 + } + + private static ArrayOf AggregateStrongKeyPrefixes( + IServiceProvider sp, + RedundancyConsistencyOptions options) + { + var prefixes = new HashSet(StringComparer.Ordinal); + ArrayOf baseSet = options.StrongKeyPrefixes.IsEmpty + ? HybridSharedKeyValueStore.DefaultStrongKeyPrefixes + : options.StrongKeyPrefixes; + foreach (string prefix in baseSet) + { + prefixes.Add(prefix); + } + foreach (IStrongKeyspaceProvider provider in sp.GetServices()) + { + foreach (string prefix in provider.GetStrongKeyPrefixes()) + { + prefixes.Add(prefix); + } + } + var result = new string[prefixes.Count]; + prefixes.CopyTo(result); + return new ArrayOf(result); + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/RedundancyConsistencyOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/RedundancyConsistencyOptions.cs new file mode 100644 index 0000000000..073ce5880a --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/RedundancyConsistencyOptions.cs @@ -0,0 +1,84 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: options for selecting the consistency model of the shared store used by a + /// RedundantServerSet (). + /// + public sealed class RedundancyConsistencyOptions + { + /// + /// Gets or sets the consistency model. (the default) serves + /// bulk state from a CRDT and complements it with Raft for the linearizable primitives; + /// serves all state from Raft. + /// + public RedundancyConsistencyMode Mode { get; set; } = RedundancyConsistencyMode.Eventual; + + /// + /// Gets or sets this replica's Raft node id. Ignored when + /// supplies the consensus replica. + /// + public ulong NodeId { get; set; } = 1; + + /// + /// Gets or sets a factory that creates the Raft consensus replica. When null, a single-node + /// is used (suitable for single-process deployments and tests; the + /// external multi-node Raft engine plugs in here). + /// + public Func? RaftConsensusFactory { get; set; } + + /// + /// Gets or sets a factory that creates the eventually-consistent bulk store used in + /// mode (typically a ). + /// When null, a singleton is used. + /// + public Func? BulkStoreFactory { get; set; } + + /// + /// Gets or sets the key prefixes routed to the linearizable Raft store in + /// mode. When empty, the defaults + /// (nonce/, lease/, election/) are used. + /// + public ArrayOf StrongKeyPrefixes { get; set; } + + /// + /// Gets or sets a value indicating whether the is registered as a native + /// (the default). Set to false to keep a separately-registered + /// election (for example the lease-based ). + /// + public bool UseRaftLeaderElection { get; set; } = true; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/ReplicatedGossipOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/ReplicatedGossipOptions.cs new file mode 100644 index 0000000000..ccc735c52b --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/ReplicatedGossipOptions.cs @@ -0,0 +1,249 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Net; +using Crdt; +using Crdt.Transport; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: shared CRDT gossip configuration: replica identity, time source, the + /// transport that disseminates state between replicas, and decoding limits. + /// + public abstract class ReplicatedGossipOptions + { + /// + /// Gets or sets this replica's stable CRDT identity. Defaults to a new + /// random identity; supply a stable value per replica in production. + /// + public ReplicaId ReplicaId { get; set; } = ReplicaId.New(); + + /// + /// Gets or sets the time source used by the logical clock. + /// + public TimeProvider TimeProvider { get; set; } = TimeProvider.System; + + /// + /// Gets or sets the factory that creates the gossip transport. When + /// null, an isolated in-process transport is used (single-process + /// / development only — no cross-node replication); configure + /// or for a real + /// deployment. + /// + public Func? TransportFactory { get; set; } + + /// + /// Gets or sets a value indicating whether a custom authenticates peers. + /// + /// + /// This only applies when assigning directly. The built-in + /// helper sets the authentication state from its mutual-TLS options. + /// + public bool TransportFactoryProvidesAuthenticatedGossip { get; set; } + + /// + /// Gets or sets a value indicating whether network gossip may start without authenticated transport. + /// + /// + /// Leave this at the secure default (false) for production. Set it to true only for + /// isolated development or test fabrics where forged CRDT frames cannot be injected by another host. + /// TCP gossip is considered authenticated when mutual TLS is configured; UDP gossip has no built-in + /// peer authentication and therefore requires this explicit opt-out. + /// + public bool AllowUnauthenticatedGossip { get; set; } + + /// + /// Gets or sets the maximum number of replicated map entries accepted + /// when decoding received state. + /// + public int MaxEntryCount { get; set; } = 1_000_000; + + /// + /// Gets or sets the maximum encoded key/payload size (bytes) accepted + /// when decoding received state. + /// + public int MaxPayloadBytes { get; set; } = 16 * 1024 * 1024; + + /// + /// Extension beyond OPC 10000-4 §6.6: configures a TCP gossip transport. Peers added via + /// are attached to the created transport. + /// + /// The local bind address. + /// The local bind port (0 for an OS-assigned port). + /// Optional anti-entropy gossip interval. + /// Optional TLS / mutual-TLS configuration. Mutual TLS is required unless + /// is explicitly enabled. + public void UseTcpGossip( + IPAddress address, + int port, + TimeSpan? gossipInterval = null, + GossipTlsOptions? tls = null) + { + if (address == null) + { + throw new ArgumentNullException(nameof(address)); + } + + TransportFactory = _ => + { + var transportOptions = new TcpGossipTransportOptions + { + Address = address, + Port = port, + Tls = tls + }; + if (gossipInterval.HasValue) + { + transportOptions.GossipInterval = gossipInterval.Value; + } + + var transport = new TcpGossipTransport(transportOptions); + transport.AddPeers(m_peers); + return transport; + }; + m_transportSecurityMode = IsMutualTlsConfigured(tls) ? + GossipTransportSecurityMode.AuthenticatedNetwork : + GossipTransportSecurityMode.UnauthenticatedNetwork; + } + + /// + /// Extension beyond OPC 10000-4 §6.6: configures a UDP datagram gossip transport. Peers added via + /// are attached to the created transport. + /// + /// The local bind address. + /// The local bind port (0 for an OS-assigned port). + /// Optional anti-entropy gossip interval. + public void UseUdpGossip(IPAddress address, int port, TimeSpan? gossipInterval = null) + { + if (address == null) + { + throw new ArgumentNullException(nameof(address)); + } + + TransportFactory = _ => + { + var transport = new UdpGossipTransport( + address, port, gossipInterval ?? TimeSpan.FromMilliseconds(500)); + transport.AddPeers(m_peers); + return transport; + }; + m_transportSecurityMode = GossipTransportSecurityMode.UnauthenticatedNetwork; + } + + /// + /// Extension beyond OPC 10000-4 §6.6: adds a peer endpoint to gossip with. Applied to the transport + /// created by / . + /// + /// The peer endpoint. + public void AddPeer(IPEndPoint endpoint) + { + if (endpoint == null) + { + throw new ArgumentNullException(nameof(endpoint)); + } + m_peers.Add(endpoint); + } + + /// + /// Builds the decoding limits for received state. + /// + internal CrdtReaderOptions CreateReaderOptions() + { + return new CrdtReaderOptions + { + MaxCollectionCount = MaxEntryCount, + MaxStringBytes = MaxPayloadBytes, + MaxDepth = CrdtReaderOptions.Default.MaxDepth + }; + } + + /// + /// Creates the transport for one replica. When no factory is + /// configured, an isolated in-process network is created and returned + /// via so the caller can dispose it. + /// + internal ITransport CreateTransport(IServiceProvider services, out InMemoryNetwork? defaultNetwork) + { + if (TransportFactory != null) + { + if (TransportFactoryProvidesAuthenticatedGossip) + { + m_transportSecurityMode = GossipTransportSecurityMode.AuthenticatedNetwork; + } + ThrowIfUnauthenticatedNetworkGossip(); + defaultNetwork = null; + return TransportFactory(services); + } + + var network = new InMemoryNetwork(); + defaultNetwork = network; + return network.CreateTransport(); + } + + private void ThrowIfUnauthenticatedNetworkGossip() + { + if (m_transportSecurityMode == GossipTransportSecurityMode.AuthenticatedNetwork || + AllowUnauthenticatedGossip) + { + return; + } + + throw new InvalidOperationException( + "CRDT network gossip is configured without authenticated transport. Address-space CRDT entries " + + "are last-writer-wins; an unauthenticated peer can forge a higher-clock frame and replace values " + + "served to clients. Configure TCP gossip with mutual TLS (server certificate, required client " + + "certificates, client certificate, and remote certificate validation), or explicitly set " + + $"{nameof(TransportFactoryProvidesAuthenticatedGossip)} for authenticated custom transports, or set " + + $"{nameof(AllowUnauthenticatedGossip)} to true for isolated development/test fabrics."); + } + + private static bool IsMutualTlsConfigured(GossipTlsOptions? tls) + { + return tls?.ServerCertificate != null && + tls.RequireClientCertificate && + tls.ClientCertificates != null && + tls.ClientCertificates.Count > 0 && + tls.RemoteCertificateValidationCallback != null; + } + + private enum GossipTransportSecurityMode + { + InProcess, + AuthenticatedNetwork, + UnauthenticatedNetwork + } + + private readonly List m_peers = []; + private GossipTransportSecurityMode m_transportSecurityMode = GossipTransportSecurityMode.InProcess; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/ReplicatedServerBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/ReplicatedServerBuilderExtensions.cs new file mode 100644 index 0000000000..d5d2be5292 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/ReplicatedServerBuilderExtensions.cs @@ -0,0 +1,113 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: fluent registration of CRDT active/active replication features on the + /// . + /// + public static class ReplicatedServerBuilderExtensions + { + /// + /// Extension beyond OPC 10000-4 §6.6: registers active/active (multi-writer) replication of the address + /// space using CRDTs gossiped between replicas. Every replica accepts + /// writes and converges without a leader; this is an alternative to the + /// leader-write active/passive UseDistributedAddressSpace. + /// Configure mutual TLS on the TCP gossip transport when address-space updates cross a network + /// boundary. Startup fails closed for unauthenticated TCP/UDP gossip unless the deployment explicitly + /// sets for an isolated test fabric. + /// + /// The server builder. + /// Optional CRDT address-space options. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseReplicatedAddressSpace( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new ReplicatedAddressSpaceOptions(); + configure?.Invoke(options); + + builder.Services.AddSingleton( + sp => new CrdtAddressSpaceStartupTask(sp, options)); + + return builder; + } + + /// + /// Extension beyond OPC 10000-4 §6.6: registers active/active session replication. Mirrored session + /// entries are gossiped between replicas as a CRDT so a client can fail + /// over to any replica and fast-reconnect. The single-use server nonce + /// stays on a strongly-consistent store (resolved from the container), + /// so the cross-replica replay defence is preserved; the authentication + /// token is never an authenticator on its own. + /// + /// + /// Mirrored session entries contain session nonces and secret material. + /// Register an to provide at-rest + /// confidentiality and integrity for those CRDT records; startup fails + /// closed without one. GossipTlsOptions protects gossip + /// traffic in transit but does not replace the record protector. + /// + /// The server builder. + /// Optional CRDT session options. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseReplicatedSessions( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new ReplicatedSessionOptions(); + configure?.Invoke(options); + + builder.Services.TryAddSingleton( + sp => new CrdtSessionManagerFactory(sp, options)); + + return builder; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Security/RecordProtectionGuard.cs b/Libraries/Opc.Ua.Redundancy.Server/Security/RecordProtectionGuard.cs new file mode 100644 index 0000000000..2a901148b2 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Security/RecordProtectionGuard.cs @@ -0,0 +1,136 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Microsoft.Extensions.DependencyInjection; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Fail-closed resolution of the that + /// protects mirrored distributed state (session entries, subscription + /// definitions, retransmission queues and continuation-point envelopes). + /// + /// + /// This protects extension state beyond OPC 10000-4 §6.6. Mirrored records carry session secrets + /// (server/client nonces), user + /// identity tokens and previously-sent notifications. When the shared + /// backend is an external, network-reachable store, writing those records + /// without authenticated encryption would expose secrets and allow a party + /// with store access to forge records. This guard therefore refuses to mirror + /// state to a non in-memory store unless a protector is configured, instead + /// of silently falling back to the no-op . + /// A deployment that knowingly accepts unprotected storage (for example a + /// throwaway test) can register explicitly + /// as an auditable opt-in. + /// + internal static class RecordProtectionGuard + { + /// + /// Resolves the registered , throwing when + /// mirrored state is backed by an external (non in-memory) shared store + /// but no protector is configured. + /// + /// The service provider. + /// + /// The registered protector, or null when an in-memory store is + /// used and no protector is configured (the store then applies the no-op + /// protector, which is safe for in-process state). + /// + /// + /// is null. + /// + /// + /// An external shared store is configured without an + /// . + /// + public static IRecordProtector? ResolveProtectorOrThrow(IServiceProvider services) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + + ISharedKeyValueStore store = services.GetRequiredService(); + return ResolveProtectorOrThrow(services, store.GetType()); + } + + /// + /// Resolves the registered for a known + /// mirrored store type. + /// + /// The service provider. + /// The mirrored store type. + /// + /// The registered protector, or null when the store type is an + /// in-memory store and no protector is configured. + /// + /// + /// or is null. + /// + /// + /// The mirrored store type is external and no is registered. + /// + internal static IRecordProtector? ResolveProtectorOrThrow( + IServiceProvider services, + Type storeType) + { + if (services == null) + { + throw new ArgumentNullException(nameof(services)); + } + if (storeType == null) + { + throw new ArgumentNullException(nameof(storeType)); + } + + IRecordProtector? protector = services.GetService(); + if (protector != null) + { + return protector; + } + + if (!typeof(InMemorySharedKeyValueStore).IsAssignableFrom(storeType)) + { + throw new InvalidOperationException( + $"Distributed state mirroring is configured with an external shared key/value store " + + $"('{storeType.Name}') but no IRecordProtector is registered. Mirrored records " + + "contain session secrets, user identity tokens and notifications; without a protector they " + + "would be written with neither confidentiality nor integrity protection. Configure a record " + + "protector (for example DistributedAddressSpaceOptions.RecordProtectorFactory returning an " + + "AesCbcHmacRecordProtector with a managed key, or register an IRecordProtector in DI). To " + + "knowingly accept unprotected storage, register NullRecordProtector explicitly."); + } + + return null; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/CrdtSessionManagerFactory.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/CrdtSessionManagerFactory.cs new file mode 100644 index 0000000000..f53a80cb1a --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/CrdtSessionManagerFactory.cs @@ -0,0 +1,177 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Crdt.Transport; +using Microsoft.Extensions.DependencyInjection; +using Opc.Ua.Security.Certificates; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: an that builds a + /// whose mirrored session entries + /// are replicated active/active by a + /// (gossip). The single-use server nonce stays on a strongly-consistent + /// store resolved from the container, preserving cross-replica replay + /// defence. + /// + /// + /// CRDT session mirroring stores records in + /// a gossip-replicated store. Those records carry session nonces and secret + /// material, so startup fails closed unless an + /// is registered. protects the transport in + /// transit; the record protector is still required for at-rest + /// confidentiality and integrity. + /// + public sealed class CrdtSessionManagerFactory : ISessionManagerFactory, IAsyncDisposable + { + /// + /// Creates the factory. + /// + /// The application service provider. + /// The CRDT session options. + public CrdtSessionManagerFactory(IServiceProvider services, ReplicatedSessionOptions options) + { + m_services = services ?? throw new ArgumentNullException(nameof(services)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + } + + /// + public ISessionManager Create( + IServerInternal server, + ApplicationConfiguration configuration, + TimeProvider timeProvider, + Func serverCertificateProvider) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + ITransport transport = m_options.CreateTransport(m_services, out InMemoryNetwork? defaultNetwork); + var entryStore = new CrdtSharedKeyValueStore( + m_options.ReplicaId, transport, m_options.TimeProvider, m_options.CreateReaderOptions()); + + IRecordProtector? protector = RecordProtectionGuard.ResolveProtectorOrThrow( + m_services, + typeof(CrdtSharedKeyValueStore)); + var sessionStore = new SharedKeyValueSessionStore(entryStore, server.MessageContext, protector); + + // The single-use nonce must be strongly consistent — never the CRDT + // store (whose CompareAndSwapAsync is not supported) and never a + // per-process fallback when fast reconnect is enabled, otherwise the + // cross-replica session-restore replay defence is silently defeated. + ISharedKeyValueStore? nonceStore = m_services.GetService(); + InMemorySharedKeyValueStore? ownedNonceStore = null; + if (nonceStore == null) + { + if (m_options.Session.EnableFastReconnect) + { + throw new InvalidOperationException( + "CRDT active/active fast reconnect requires a strongly-consistent, cross-replica " + + "ISharedKeyValueStore for the single-use server-nonce registry. The eventually-" + + "consistent CRDT gossip store cannot enforce single-use (CompareAndSwapAsync is not " + + "supported), so without a strongly-consistent registry shared by every replica the " + + "cross-replica session-restore replay defence is defeated. Register a strongly-" + + "consistent backend (for example a Redis ISharedKeyValueStore adapter) shared across " + + "the replica set, or disable DistributedSessionOptions.EnableFastReconnect."); + } + + ownedNonceStore = new InMemorySharedKeyValueStore(); + nonceStore = ownedNonceStore; + } + var nonceRegistry = new SharedSingleUseNonceRegistry(nonceStore); + + lock (m_lock) + { + m_entryStores.Add(entryStore); + if (defaultNetwork != null) + { + m_defaultNetworks.Add(defaultNetwork); + } + if (ownedNonceStore != null) + { + m_ownedNonceStores.Add(ownedNonceStore); + } + } + + return new DistributedSessionManager( + server, + configuration, + sessionStore, + nonceRegistry, + serverCertificateProvider, + m_options.Session, + timeProvider); + } + + /// + public async ValueTask DisposeAsync() + { + CrdtSharedKeyValueStore[] stores; + InMemoryNetwork[] networks; + InMemorySharedKeyValueStore[] nonceStores; + lock (m_lock) + { + stores = [.. m_entryStores]; + m_entryStores.Clear(); + networks = [.. m_defaultNetworks]; + m_defaultNetworks.Clear(); + nonceStores = [.. m_ownedNonceStores]; + m_ownedNonceStores.Clear(); + } + + foreach (CrdtSharedKeyValueStore store in stores) + { + await store.DisposeAsync().ConfigureAwait(false); + } + foreach (InMemoryNetwork network in networks) + { + await network.DisposeAsync().ConfigureAwait(false); + } + foreach (InMemorySharedKeyValueStore nonceStore in nonceStores) + { + nonceStore.Dispose(); + } + } + + private readonly IServiceProvider m_services; + private readonly ReplicatedSessionOptions m_options; + private readonly Lock m_lock = new(); + private readonly List m_entryStores = []; + private readonly List m_defaultNetworks = []; + private readonly List m_ownedNonceStores = []; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs new file mode 100644 index 0000000000..c46b90ad8a --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManager.cs @@ -0,0 +1,479 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Security.Certificates; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// A that mirrors session state to a shared + /// store so a client can fail over to a standby replica and reconnect by + /// re-running ActivateSession on a new SecureChannel — the OPC UA + /// HotAndMirrored fast reconnect (Part 4 §6.6). + /// + /// + /// + /// Security model (see Docs/HighAvailability.md): the + /// AuthenticationToken is only a lookup key. On restore the standard + /// activation path still performs the full client-certificate signature + /// check against the mirrored serverNonce, the nonce is consumed + /// exactly once across the replica set (so a captured activation cannot be + /// replayed), and the SecurityPolicy/Mode must match the original. + /// The mirrored record is encrypted and integrity-protected at + /// rest by the 's record protector. + /// + /// + /// The safe default is re-authentication on failover + /// ( = false): + /// the manager mirrors metadata for visibility but does not admit a session + /// from the shared store. + /// + /// + public sealed class DistributedSessionManager : SessionManager + { + /// + /// Creates a distributed session manager. + /// + /// The hosting server. + /// The application configuration. + /// The shared session store (encrypted). + /// The cross-replica single-use nonce registry. + /// + /// Resolves the shared server ApplicationInstanceCertificate for a + /// security policy URI. The returned certificate must be + /// long-lived (owned by the caller); the session does not dispose it. + /// + /// The distributed session options. + /// An optional time provider. + public DistributedSessionManager( + IServerInternal server, + ApplicationConfiguration configuration, + ISharedSessionStore sessionStore, + ISingleUseNonceRegistry nonceRegistry, + Func serverCertificateProvider, + DistributedSessionOptions? options = null, + TimeProvider? timeProvider = null) + : base(server, configuration, timeProvider) + { + m_server = server ?? throw new ArgumentNullException(nameof(server)); + m_sessionStore = sessionStore ?? throw new ArgumentNullException(nameof(sessionStore)); + m_nonceRegistry = nonceRegistry ?? throw new ArgumentNullException(nameof(nonceRegistry)); + m_serverCertificateProvider = serverCertificateProvider + ?? throw new ArgumentNullException(nameof(serverCertificateProvider)); + m_options = options ?? new DistributedSessionOptions(); + m_telemetry = server.Telemetry; + m_logger = server.Telemetry.CreateLogger(); + } + + /// + protected override bool SupportsSessionRestore => m_options.EnableFastReconnect; + + /// + public override async ValueTask CreateSessionAsync( + OperationContext context, + Certificate serverCertificate, + string? sessionName, + ByteString clientNonce, + ApplicationDescription? clientDescription, + string? endpointUrl, + Certificate? clientCertificate, + CertificateCollection? clientCertificateChain, + double requestedSessionTimeout, + uint maxResponseMessageSize, + CancellationToken cancellationToken = default) + { + CreateSessionResult result = await base.CreateSessionAsync( + context, + serverCertificate, + sessionName, + clientNonce, + clientDescription, + endpointUrl, + clientCertificate, + clientCertificateChain, + requestedSessionTimeout, + maxResponseMessageSize, + cancellationToken).ConfigureAwait(false); + + try + { + m_tokensBySession[result.SessionId] = result.AuthenticationToken; + SharedSessionEntry entry = BuildEntry( + context, + result, + sessionName, + clientNonce, + clientDescription, + endpointUrl, + clientCertificate); + await m_sessionStore.PutAsync(entry, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + // Mirroring is best-effort; never fail an otherwise-valid session + // because the shared store is unavailable. + m_logger.LogWarning(ex, "Failed to mirror created session to the shared store."); + } + + return result; + } + + /// + public override async ValueTask<(bool IdentityContextChanged, ByteString ServerNonce, ServiceResult ActivationStatus)> ActivateSessionAsync( + OperationContext context, + NodeId authenticationToken, + SignatureData? clientSignature, + ExtensionObject userIdentityToken, + SignatureData? userTokenSignature, + ArrayOf localeIds, + CancellationToken cancellationToken = default) + { + (bool IdentityContextChanged, ByteString ServerNonce, ServiceResult ActivationStatus) result = + await base.ActivateSessionAsync( + context, + authenticationToken, + clientSignature, + userIdentityToken, + userTokenSignature, + localeIds, + cancellationToken).ConfigureAwait(false); + + // Mirror the freshly issued serverNonce so a standby validates the + // client's next activation against it (and consumes it single-use). + try + { + await MirrorActivationAsync(authenticationToken, result.ServerNonce, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogWarning(ex, "Failed to mirror activated session to the shared store."); + } + + return result; + } + + /// + public override async ValueTask CloseSessionAsync( + NodeId sessionId, + CancellationToken cancellationToken = default) + { + if (m_tokensBySession.TryRemove(sessionId, out NodeId token)) + { + try + { + await m_sessionStore.RemoveAsync(token, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger.LogWarning(ex, "Failed to remove mirrored session from the shared store."); + } + } + + await base.CloseSessionAsync(sessionId, cancellationToken).ConfigureAwait(false); + } + + /// + protected override async ValueTask RestoreSessionAsync( + NodeId authenticationToken, + OperationContext context, + CancellationToken cancellationToken = default) + { + if (!m_options.EnableFastReconnect) + { + return null; + } + + SharedSessionEntry? entry = await m_sessionStore + .TryGetAsync(authenticationToken, cancellationToken) + .ConfigureAwait(false); + if (entry == null) + { + return null; + } + + EndpointDescription endpoint = context.ChannelContext!.EndpointDescription!; + RestoreDecision decision = await AuthorizeAndConsumeAsync( + entry, + endpoint.SecurityPolicyUri ?? string.Empty, + endpoint.SecurityMode, + cancellationToken).ConfigureAwait(false); + if (decision != RestoreDecision.Authorized) + { + m_logger.LogWarning( + "Distributed session restore for {Token} rejected: {Reason}.", + TokenDigest(authenticationToken), + decision); + return null; + } + + ISession? session = ReconstructSession(entry, authenticationToken, context); + if (session != null) + { + try + { + await session.InitializeAsync(context, cancellationToken).ConfigureAwait(false); + if (session is Session restoredSession) + { + await restoredSession + .LoadMirroredContinuationPointsAsync(entry.SessionId, cancellationToken) + .ConfigureAwait(false); + } + } + catch + { + session.Dispose(); + throw; + } + m_logger.LogInformation( + "Distributed session restored from shared store for {Token} (session {SessionId}).", + TokenDigest(authenticationToken), + session.Id); + m_tokensBySession[session.Id] = authenticationToken; + + // Security-relevant provenance: a session materialized on this + // replica from shared state (audit, not just a log). + m_server.ReportAuditSessionRestoredEvent( + TokenDigest(authenticationToken), session, m_logger); + } + return session; + } + + /// + /// The result of authorizing a session restore. + /// + internal enum RestoreDecision + { + /// The restore is authorized and the nonce was consumed. + Authorized, + + /// The SecurityPolicy/Mode does not match the original. + PolicyMismatch, + + /// The mirrored nonce is missing or was already consumed (replay). + NonceReplayed + } + + /// + /// Enforces the cross-channel security checks and consumes the mirrored + /// serverNonce exactly once. Factored out so the security policy + /// is unit-testable without constructing a live session. + /// + internal async ValueTask AuthorizeAndConsumeAsync( + SharedSessionEntry entry, + string securityPolicyUri, + MessageSecurityMode securityMode, + CancellationToken cancellationToken = default) + { + if (!string.Equals(entry.SecurityPolicyUri, securityPolicyUri, StringComparison.Ordinal) || + entry.SecurityMode != (int)securityMode) + { + return RestoreDecision.PolicyMismatch; + } + + if (entry.ServerNonce.IsNull || entry.ServerNonce.IsEmpty) + { + return RestoreDecision.NonceReplayed; + } + + bool consumed = await m_nonceRegistry + .TryConsumeAsync(entry.ServerNonce, cancellationToken) + .ConfigureAwait(false); + return consumed ? RestoreDecision.Authorized : RestoreDecision.NonceReplayed; + } + + private ISession? ReconstructSession( + SharedSessionEntry entry, + NodeId authenticationToken, + OperationContext context) + { + bool requiresClientCertificate = + entry.SecurityMode != (int)MessageSecurityMode.None || + !string.Equals(entry.SecurityPolicyUri, SecurityPolicies.None, StringComparison.Ordinal); + if (requiresClientCertificate && + (entry.ClientCertificateChain.IsNull || entry.ClientCertificateChain.IsEmpty)) + { + m_logger.LogWarning("Mirrored session has no client certificate; cannot restore."); + return null; + } + + Certificate? serverCertificate = m_serverCertificateProvider(entry.SecurityPolicyUri); + if (serverCertificate == null) + { + m_logger.LogWarning( + "No server certificate available for policy {Policy}; cannot restore session.", + entry.SecurityPolicyUri); + return null; + } + + using CertificateCollection parsed = entry.ClientCertificateChain.IsNull || + entry.ClientCertificateChain.IsEmpty + ? [] + : Utils.ParseCertificateChainBlob(entry.ClientCertificateChain.ToArray(), m_telemetry); + if (requiresClientCertificate && parsed.Count == 0) + { + return null; + } + + // The session owns and disposes the client certificate + issuers and + // the server nonce once created, so hand it independent (ref-counted) + // handles and dispose them only if creation fails (assign null after + // ownership transfer, per the CA2000 pattern). Trust of the chain was + // already established when the session was created on the active. + Certificate? clientCertificate = null; + CertificateCollection? issuers = null; + Nonce? serverNonce = null; + try + { + clientCertificate = parsed.Count == 0 ? null : parsed[0].AddRef(); + issuers = new CertificateCollection(); + serverNonce = Nonce.CreateNonce(entry.SecurityPolicyUri, entry.ServerNonce.ToArray()); + ISession session = CreateSession( + context, + m_server, + serverCertificate, + authenticationToken, + entry.ClientNonce, + serverNonce, + entry.SessionName, + entry.ClientDescription, + entry.EndpointUrl, + clientCertificate!, + issuers, + entry.SessionTimeout, + 0, + 0, + 0); + + // Ownership transferred to the session; prevent the finally below + // from disposing handles the session now manages. + clientCertificate = null; + issuers = null; + serverNonce = null; + return session; + } + finally + { + clientCertificate?.Dispose(); + issuers?.Dispose(); + serverNonce?.Dispose(); + } + } + + private SharedSessionEntry BuildEntry( + OperationContext context, + CreateSessionResult result, + string? sessionName, + ByteString clientNonce, + ApplicationDescription? clientDescription, + string? endpointUrl, + Certificate? clientCertificate) + { + EndpointDescription endpoint = context.ChannelContext!.EndpointDescription!; + ByteString clientCertBlob = default; + if (clientCertificate != null) + { + using var leaf = new CertificateCollection { clientCertificate }; + clientCertBlob = ByteString.From(Utils.CreateCertificateChainBlob(leaf)); + } + + return new SharedSessionEntry + { + SessionId = result.SessionId, + AuthenticationToken = result.AuthenticationToken, + SessionName = sessionName ?? string.Empty, + CreatedAt = DateTimeUtc.Now, + LastActivatedAt = DateTimeUtc.Now, + ServerNonce = result.ServerNonce, + ClientNonce = clientNonce, + ClientCertificateChain = clientCertBlob, + SecurityPolicyUri = endpoint.SecurityPolicyUri ?? string.Empty, + SecurityMode = (int)endpoint.SecurityMode, + EndpointUrl = endpointUrl ?? string.Empty, + SessionTimeout = result.RevisedSessionTimeout, + ClientDescription = clientDescription ?? new ApplicationDescription() + }; + } + + private async ValueTask MirrorActivationAsync( + NodeId authenticationToken, + ByteString serverNonce, + CancellationToken cancellationToken) + { + SharedSessionEntry? existing = await m_sessionStore + .TryGetAsync(authenticationToken, cancellationToken) + .ConfigureAwait(false); + if (existing == null) + { + return; + } + + SharedSessionEntry updated = existing with + { + ServerNonce = serverNonce, + LastActivatedAt = DateTimeUtc.Now + }; + await m_sessionStore.PutAsync(updated, cancellationToken).ConfigureAwait(false); + } + + private static string TokenDigest(NodeId authenticationToken) + { + byte[] data = Encoding.UTF8.GetBytes(authenticationToken.ToString()); +#if NET8_0_OR_GREATER + byte[] hash = SHA256.HashData(data); +#else + byte[] hash; + using (var sha = SHA256.Create()) + { + hash = sha.ComputeHash(data); + } +#endif + return Convert.ToBase64String(hash, 0, 8); + } + + private readonly IServerInternal m_server; + private readonly ISharedSessionStore m_sessionStore; + private readonly ISingleUseNonceRegistry m_nonceRegistry; + private readonly Func m_serverCertificateProvider; + private readonly DistributedSessionOptions m_options; + private readonly ITelemetryContext m_telemetry; + private readonly ILogger m_logger; + private readonly ConcurrentDictionary m_tokensBySession = new(); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManagerFactory.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManagerFactory.cs new file mode 100644 index 0000000000..16cccbe480 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionManagerFactory.cs @@ -0,0 +1,95 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Opc.Ua.Security.Certificates; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// An that builds a + /// over a shared key/value store. + /// The session store and nonce registry are created lazily in + /// , when the server's populated message context is + /// available. + /// + public sealed class DistributedSessionManagerFactory : ISessionManagerFactory + { + /// + /// Creates the factory. + /// + /// The shared key/value backend. + /// + /// Optional record protector applied to every mirrored session entry + /// (authenticated encryption); defaults to a no-op pass-through. + /// + /// The distributed session options. + public DistributedSessionManagerFactory( + ISharedKeyValueStore keyValueStore, + IRecordProtector? protector = null, + DistributedSessionOptions? options = null) + { + m_keyValueStore = keyValueStore ?? throw new ArgumentNullException(nameof(keyValueStore)); + m_protector = protector; + m_options = options ?? new DistributedSessionOptions(); + } + + /// + public ISessionManager Create( + IServerInternal server, + ApplicationConfiguration configuration, + TimeProvider timeProvider, + Func serverCertificateProvider) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + var sessionStore = new SharedKeyValueSessionStore( + m_keyValueStore, server.MessageContext, m_protector); + var nonceRegistry = new SharedSingleUseNonceRegistry(m_keyValueStore); + + return new DistributedSessionManager( + server, + configuration, + sessionStore, + nonceRegistry, + serverCertificateProvider, + m_options, + timeProvider); + } + + private readonly ISharedKeyValueStore m_keyValueStore; + private readonly IRecordProtector? m_protector; + private readonly DistributedSessionOptions m_options; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionMirroringBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionMirroringBuilderExtensions.cs new file mode 100644 index 0000000000..0443c41dd6 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionMirroringBuilderExtensions.cs @@ -0,0 +1,82 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Fluent registration for cross-replica session mirroring. + /// + public static class DistributedSessionMirroringBuilderExtensions + { + /// + /// Registers distributed session mirroring for HotAndMirrored or Transparent failover. + /// + /// + /// The mirrored session record is keyed by a digest of the session authentication token and is protected + /// with the configured . Enable + /// to allow a backup replica to materialize the + /// session during ActivateSession; validation still runs through the core activation path and consumes + /// the mirrored server nonce once across the replica set. + /// + /// The server builder. + /// Optional session mirroring options. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseDistributedSessionMirroring( + this IOpcUaServerBuilder builder, + Action? configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new DistributedSessionOptions(); + configure?.Invoke(options); + + builder.Services.TryAddSingleton(_ => new InMemorySharedKeyValueStore()); + builder.Services.TryAddSingleton(sp => + new DistributedSessionManagerFactory( + sp.GetRequiredService(), + RecordProtectionGuard.ResolveProtectorOrThrow(sp), + options)); + + return builder; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionOptions.cs new file mode 100644 index 0000000000..be03e8308d --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/DistributedSessionOptions.cs @@ -0,0 +1,56 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Options controlling the . + /// + public sealed class DistributedSessionOptions + { + /// + /// Gets or sets a value indicating whether a standby replica may + /// restore a mirrored session and let a client reconnect by + /// re-running ActivateSession on a new SecureChannel (the + /// OPC UA HotAndMirrored fast reconnect). + /// + /// + /// Defaults to false — the safe, spec-compliant default is + /// re-authentication on failover (a fresh CreateSession + + /// ActivateSession), which needs no shared session state. Even + /// when this is false the manager still mirrors session metadata + /// for cross-replica visibility, but it will not admit a session from + /// the shared store. When set to true, a reconnect still performs + /// the full ActivateSession client-signature validation against + /// the mirrored, single-use serverNonce; the token is never an + /// authenticator on its own. + /// + public bool EnableFastReconnect { get; set; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/ISharedSessionStore.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/ISharedSessionStore.cs new file mode 100644 index 0000000000..bd1b61fc87 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/ISharedSessionStore.cs @@ -0,0 +1,74 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Shares session context across server replicas keyed by the session + /// AuthenticationToken, enabling fast active/passive reconnect: + /// after a failover a client re-activates against the promoted replica + /// using only its token, without a full re-handshake. + /// + /// + /// Analogous to for subscriptions. The + /// default implementation + /// () persists entries in the + /// same shared key/value backend as the distributed address space. + /// + public interface ISharedSessionStore + { + /// + /// Stores or replaces a session entry (keyed by its + /// ). + /// + /// The session entry. + /// Cancellation token. + ValueTask PutAsync(SharedSessionEntry entry, CancellationToken ct = default); + + /// + /// Looks up a session entry by authentication token. + /// + /// The session token. + /// Cancellation token. + /// The entry, or null when absent. + ValueTask TryGetAsync(NodeId authenticationToken, CancellationToken ct = default); + + /// + /// Removes a session entry (e.g. on close). + /// + /// The session token. + /// Cancellation token. + /// true when an entry was removed. + ValueTask RemoveAsync(NodeId authenticationToken, CancellationToken ct = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/ISingleUseNonceRegistry.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/ISingleUseNonceRegistry.cs new file mode 100644 index 0000000000..ae60f6f642 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/ISingleUseNonceRegistry.cs @@ -0,0 +1,64 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Records server nonces as single-use across the whole replica set so a + /// captured ActivateSession signature cannot be replayed on another + /// replica during a mirrored fast-reconnect. A nonce may be consumed + /// exactly once by exactly one replica; every later attempt (on any + /// replica) is rejected. + /// + /// + /// OPC UA Part 4 §5.7.3.1 requires the serverNonce to be single-use. + /// In a shared / mirrored session deployment that guarantee must hold across + /// replicas, otherwise a Sign-mode ActivateSession captured against + /// one replica replays against a standby. This + /// registry provides the cross-replica single-use check. + /// + public interface ISingleUseNonceRegistry + { + /// + /// Atomically marks as consumed. + /// + /// The server nonce being consumed. + /// Cancellation token. + /// + /// true when this call was the first to consume the nonce (the + /// caller may proceed); false when the nonce was already consumed + /// (the caller must reject the request as a replay). + /// + ValueTask TryConsumeAsync(ByteString nonce, CancellationToken ct = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/ReplicatedSessionOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/ReplicatedSessionOptions.cs new file mode 100644 index 0000000000..e77458ecf4 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/ReplicatedSessionOptions.cs @@ -0,0 +1,59 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: options for CRDT active/active session replication. Mirrored session + /// entries are gossiped between replicas as a CRDT; the gossip configuration + /// is inherited from . + /// + /// + /// The single-use server nonce is not replicated as a CRDT — it + /// requires a strongly-consistent compare-and-swap that CRDTs cannot + /// provide. The session manager keeps the nonce on a separate + /// strongly-consistent resolved from the + /// container (defaulting to an in-process store for development). + /// CRDT session entries themselves require an + /// because they are gossiped and contain session nonces and secret material. + /// Startup fails closed unless a protector is registered; use + /// GossipTlsOptions separately to protect the gossip transport. + /// + public sealed class ReplicatedSessionOptions : ReplicatedGossipOptions + { + /// + /// Gets the distributed session behavior (fast-reconnect opt-in). The + /// safe default re-authenticates on failover. + /// + public DistributedSessionOptions Session { get; } = new(); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedKeyValueSessionStore.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedKeyValueSessionStore.cs new file mode 100644 index 0000000000..f85afdfcf4 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedKeyValueSessionStore.cs @@ -0,0 +1,181 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Default over an + /// . Each entry is binary-encoded and + /// then passed through an so the session + /// secret material is encrypted and integrity-protected at rest; a tampered + /// or forged entry fails verification and is treated as absent (fail-closed). + /// The store key is the SHA-256 digest of the authentication token, not the + /// token itself, so the secret-bearing keyspace stays one-way (the raw token + /// is never exposed via a backend's key enumeration / monitoring / dumps). + /// + public sealed class SharedKeyValueSessionStore : ISharedSessionStore + { + /// + /// Creates a session store over a shared key/value backend. + /// + /// The shared key/value backend. + /// The message context for encoding. + /// + /// Optional record protector applied to every encoded session entry + /// (authenticated encryption); defaults to a no-op pass-through. + /// Configure an in production + /// so the shared store can be treated as untrusted. + /// + public SharedKeyValueSessionStore( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector? protector = null) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? NullRecordProtector.Instance; + } + + /// + public ValueTask PutAsync(SharedSessionEntry entry, CancellationToken ct = default) + { + if (entry == null) + { + throw new ArgumentNullException(nameof(entry)); + } + if (entry.AuthenticationToken.IsNull) + { + throw new ArgumentException( + "Session entry must have a non-null authentication token.", + nameof(entry)); + } + return m_store.SetAsync( + KeyFor(entry.AuthenticationToken), m_protector.Protect(Encode(entry)), ct); + } + + /// + public async ValueTask TryGetAsync( + NodeId authenticationToken, + CancellationToken ct = default) + { + (bool found, ByteString value) = await m_store + .TryGetAsync(KeyFor(authenticationToken), ct) + .ConfigureAwait(false); + if (found && m_protector.TryUnprotect(value, out ByteString payload)) + { + return Decode(payload); + } + return null; + } + + /// + public ValueTask RemoveAsync(NodeId authenticationToken, CancellationToken ct = default) + { + return m_store.DeleteAsync(KeyFor(authenticationToken), ct); + } + + /// + /// Computes the shared-store key for an authentication token: the + /// configured prefix followed by the SHA-256 digest of the token, so the + /// raw token never appears in the keyspace. + /// + /// The session authentication token. + /// The opaque store key. + internal static string KeyFor(NodeId authenticationToken) + { + byte[] data = Encoding.UTF8.GetBytes(authenticationToken.ToString()); +#if NET8_0_OR_GREATER + byte[] hash = SHA256.HashData(data); +#else + byte[] hash; + using (var sha = SHA256.Create()) + { + hash = sha.ComputeHash(data); + } +#endif + return Prefix + Convert.ToBase64String(hash); + } + + private ByteString Encode(SharedSessionEntry entry) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteNodeId(null, entry.SessionId); + encoder.WriteNodeId(null, entry.AuthenticationToken); + encoder.WriteString(null, entry.SessionName); + encoder.WriteInt64(null, entry.CreatedAt); + encoder.WriteInt64(null, entry.LastActivatedAt); + encoder.WriteByteString(null, entry.ServerNonce); + encoder.WriteByteString(null, entry.ClientNonce); + encoder.WriteByteString(null, entry.ClientCertificateChain); + encoder.WriteString(null, entry.SecurityPolicyUri); + encoder.WriteInt32(null, entry.SecurityMode); + encoder.WriteString(null, entry.EndpointUrl); + encoder.WriteDouble(null, entry.SessionTimeout); + encoder.WriteEncodeable(null, entry.ClientDescription ?? new ApplicationDescription()); + encoder.WriteByteString(null, entry.SecretMaterial); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : new ByteString(buffer); + } + + private SharedSessionEntry Decode(ByteString payload) + { + using var decoder = new BinaryDecoder(payload.ToArray(), m_context); + return new SharedSessionEntry + { + SessionId = decoder.ReadNodeId(null), + AuthenticationToken = decoder.ReadNodeId(null), + SessionName = decoder.ReadString(null) ?? string.Empty, + CreatedAt = decoder.ReadInt64(null), + LastActivatedAt = decoder.ReadInt64(null), + ServerNonce = decoder.ReadByteString(null), + ClientNonce = decoder.ReadByteString(null), + ClientCertificateChain = decoder.ReadByteString(null), + SecurityPolicyUri = decoder.ReadString(null) ?? string.Empty, + SecurityMode = decoder.ReadInt32(null), + EndpointUrl = decoder.ReadString(null) ?? string.Empty, + SessionTimeout = decoder.ReadDouble(null), + ClientDescription = decoder.ReadEncodeable(null), + SecretMaterial = decoder.ReadByteString(null) + }; + } + + private const string Prefix = "session/"; + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedSessionEntry.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedSessionEntry.cs new file mode 100644 index 0000000000..a1a8398f01 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedSessionEntry.cs @@ -0,0 +1,133 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// The session context shared across replicas so a standby can + /// re-activate a session after a failover. The token is a lookup key + /// only; admission still requires a full ActivateSession signature + /// check against the (single-use) on the standby. + /// + /// + /// The whole entry is encrypted and integrity-protected at rest by the + /// configured before it reaches the shared + /// store, so the secret-bearing fields (, + /// ) are never written in cleartext. Certificate + /// stores are assumed to be shared independently; the shared + /// ApplicationInstanceCertificate is supplied by the + /// server, not this entry. + /// + public sealed record SharedSessionEntry + { + /// + /// The server-assigned session identifier. + /// + public NodeId SessionId { get; init; } = NodeId.Null; + + /// + /// The authentication token used as the lookup key on reconnect. + /// + public NodeId AuthenticationToken { get; init; } = NodeId.Null; + + /// + /// The session name. + /// + public string SessionName { get; init; } = string.Empty; + + /// + /// When the session was created (UTC). + /// + public DateTimeUtc CreatedAt { get; init; } + + /// + /// When the session was last activated (UTC). + /// + public DateTimeUtc LastActivatedAt { get; init; } + + /// + /// The last serverNonce issued for the session — the value the + /// client signs on its next ActivateSession. Single-use: a + /// standby must invalidate it (via an + /// ) when it consumes it on + /// restore so a captured activation cannot be replayed. + /// + public ByteString ServerNonce { get; init; } + + /// + /// The client nonce associated with the session. + /// + public ByteString ClientNonce { get; init; } + + /// + /// The client certificate chain (leaf first, then issuers) as a single + /// blob (see ). Used to + /// reconstruct the session and to enforce that a failover reconnect + /// presents the same client certificate. + /// + public ByteString ClientCertificateChain { get; init; } + + /// + /// The security policy URI in force for the session — needed to rebuild + /// the typed serverNonce and to enforce that a failover reconnect + /// uses the same SecurityPolicy. + /// + public string SecurityPolicyUri { get; init; } = string.Empty; + + /// + /// The message security mode (cast of ) + /// in force for the session. + /// + public int SecurityMode { get; init; } + + /// + /// The endpoint URL the session was created against. + /// + public string EndpointUrl { get; init; } = string.Empty; + + /// + /// The revised session timeout, in milliseconds. + /// + public double SessionTimeout { get; init; } + + /// + /// The client application description supplied at session creation. + /// + public ApplicationDescription ClientDescription { get; init; } = new(); + + /// + /// Optional opaque, caller-encrypted secret material. May be a null + /// . + /// + public ByteString SecretMaterial { get; init; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedSingleUseNonceRegistry.cs b/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedSingleUseNonceRegistry.cs new file mode 100644 index 0000000000..81b51f22c7 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Sessions/SharedSingleUseNonceRegistry.cs @@ -0,0 +1,98 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// backed by an + /// . Consumption is a single + /// compare-and-swap that creates a marker key only when it is absent, so the + /// store's atomicity guarantees exactly one replica wins the race to consume + /// a given nonce. The nonce itself is never stored: the key is the SHA-256 + /// digest of the nonce, so the secret-bearing keyspace stays one-way. + /// + public sealed class SharedSingleUseNonceRegistry : ISingleUseNonceRegistry + { + /// + /// Creates a registry over a shared key/value backend. + /// + /// The shared key/value backend. + /// + /// The key prefix under which consumed-nonce markers are recorded. + /// + public SharedSingleUseNonceRegistry(ISharedKeyValueStore store, string keyPrefix = "nonce/") + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_keyPrefix = keyPrefix ?? throw new ArgumentNullException(nameof(keyPrefix)); + } + + /// + public ValueTask TryConsumeAsync(ByteString nonce, CancellationToken ct = default) + { + if (nonce.IsNull || nonce.IsEmpty) + { + throw new ArgumentException("Nonce must not be null or empty.", nameof(nonce)); + } + + string key = m_keyPrefix + Digest(nonce); + byte[] marker = new byte[8]; + BinaryPrimitives.WriteInt64LittleEndian(marker, DateTime.UtcNow.Ticks); + + // expected = default(ByteString) => require the key to be absent, so + // the swap succeeds only for the first consumer of this nonce. + return m_store.CompareAndSwapAsync(key, default, new ByteString(marker), ct); + } + + private static string Digest(ByteString nonce) + { + byte[] data = nonce.ToArray(); +#if NET8_0_OR_GREATER + byte[] hash = SHA256.HashData(data); +#else + byte[] hash; + using (var sha = SHA256.Create()) + { + hash = sha.ComputeHash(data); + } +#endif + return Convert.ToBase64String(hash); + } + + private readonly ISharedKeyValueStore m_store; + private readonly string m_keyPrefix; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/AddressSpaceSynchronizer.cs b/Libraries/Opc.Ua.Redundancy.Server/State/AddressSpaceSynchronizer.cs new file mode 100644 index 0000000000..b28c637e7c --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/AddressSpaceSynchronizer.cs @@ -0,0 +1,437 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: default . See the interface for + /// the writer/reader role model. + /// + public sealed class AddressSpaceSynchronizer : IAddressSpaceSynchronizer + { + /// + /// Creates a synchronizer between a local graph and a shared store. + /// + /// The shared node state store. + /// The local node graph. + /// + /// Predicate that reports whether this replica is the writer + /// (leader). Defaults to always-writer (single instance). + /// + /// Optional logger for replication errors. + public AddressSpaceSynchronizer( + INodeStateStore store, + ILocalAddressSpace addressSpace, + Func? isWriter = null, + ILogger? logger = null) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_addressSpace = addressSpace ?? throw new ArgumentNullException(nameof(addressSpace)); + m_isWriter = isWriter ?? (static () => true); + m_logger = logger; + m_onChanged = OnLocalNodeChanged; + m_onNodeAdded = OnLocalNodeAdded; + m_onNodeRemoved = OnLocalNodeRemoved; + } + + /// + public bool IsWriter => m_isWriter(); + + /// + /// Raised (for tests) after each inbound change is applied. + /// + internal event Action? InboundApplied; + + /// + public async ValueTask SeedOrHydrateAsync(CancellationToken ct = default) + { + bool any = false; + await foreach (IStoredNode stored in m_store.EnumerateAsync(ct).ConfigureAwait(false)) + { + any = true; + await TryApplyUpsertAsync(stored.NodeId, stored.Payload, ct).ConfigureAwait(false); + } + + if (any) + { + // Apply the latest value for every hydrated variable; value + // keys can be newer than the node payload they were carved + // from. Stream the value keyspace in one pass instead of + // issuing a read per variable so hydrating a large address + // space costs a bounded number of round trips. + await foreach ((NodeId nodeId, DataValue value) in m_store + .EnumerateValuesAsync(ct) + .ConfigureAwait(false)) + { + ApplyValue(nodeId, value); + } + return; + } + + if (m_isWriter()) + { + foreach (NodeState node in m_addressSpace.Nodes) + { + await m_store + .UpsertNodeAsync( + new StoredNode(node.NodeId, NodeStateSerializer.Serialize(m_addressSpace.Context, node)), + ct) + .ConfigureAwait(false); + if (node is BaseVariableState variable) + { + await m_store + .WriteValueAsync( + node.NodeId, + new DataValue(variable.Value, variable.StatusCode, variable.Timestamp), + ct) + .ConfigureAwait(false); + } + } + } + } + + /// + public void Start() + { + lock (m_lock) + { + if (m_started) + { + return; + } + m_started = true; + + if (m_isWriter()) + { + m_outbound = Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + m_addressSpace.NodeAdded += m_onNodeAdded; + m_addressSpace.NodeRemoved += m_onNodeRemoved; + foreach (NodeState node in m_addressSpace.Nodes) + { + AttachStateChanged(node); + } + m_outboundTask = Task.Run(() => DrainOutboundAsync(m_cts.Token)); + } + else + { + // Register the change-feed watcher synchronously (the + // first MoveNextAsync runs the iterator prefix that adds + // the watcher) so no change published after Start() + // returns is missed, then consume it on a background task. + m_inboundEnumerator = m_store.SubscribeChangesAsync(m_cts.Token).GetAsyncEnumerator(); + ValueTask firstMove = m_inboundEnumerator.MoveNextAsync(); + m_inboundTask = Task.Run(() => ApplyInboundLoopAsync(firstMove)); + } + } + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_lock) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + + m_cts.Cancel(); + m_outbound?.Writer.TryComplete(); + + await AwaitQuietlyAsync(m_outboundTask).ConfigureAwait(false); + await AwaitQuietlyAsync(m_inboundTask).ConfigureAwait(false); + + // The inbound loop has finished; dispose the enumerator it owned. + if (m_inboundEnumerator != null) + { + await m_inboundEnumerator.DisposeAsync().ConfigureAwait(false); + m_inboundEnumerator = null; + } + + m_addressSpace.NodeAdded -= m_onNodeAdded; + m_addressSpace.NodeRemoved -= m_onNodeRemoved; + DetachAll(); + m_cts.Dispose(); + } + + private void OnLocalNodeAdded(NodeState node) + { + AttachStateChanged(node); + Enqueue(OutboundOp.ForUpsert( + node.NodeId, + NodeStateSerializer.Serialize(m_addressSpace.Context, node))); + } + + private void OnLocalNodeRemoved(NodeId nodeId) + { + Enqueue(OutboundOp.ForDelete(nodeId)); + } + + private void OnLocalNodeChanged(ISystemContext context, NodeState node, NodeStateChangeMasks changes) + { + // Deletes are driven by ILocalAddressSpace.NodeRemoved. + if ((changes & NodeStateChangeMasks.Deleted) != 0) + { + return; + } + + if (node is BaseVariableState variable && (changes & NodeStateChangeMasks.Value) != 0) + { + Enqueue(OutboundOp.ForValue( + node.NodeId, + new DataValue(variable.Value, variable.StatusCode, variable.Timestamp))); + } + + if ((changes & (NodeStateChangeMasks.NonValue | + NodeStateChangeMasks.Children | + NodeStateChangeMasks.References)) != 0) + { + Enqueue(OutboundOp.ForUpsert( + node.NodeId, + NodeStateSerializer.Serialize(m_addressSpace.Context, node))); + } + } + + private void Enqueue(OutboundOp op) + { + m_outbound?.Writer.TryWrite(op); + } + + private async Task DrainOutboundAsync(CancellationToken ct) + { + try + { + await foreach (OutboundOp op in m_outbound!.Reader.ReadAllAsync(ct).ConfigureAwait(false)) + { + try + { + switch (op.Kind) + { + case OutboundOpKind.Value: + await m_store.WriteValueAsync(op.NodeId, op.Value, ct).ConfigureAwait(false); + break; + case OutboundOpKind.Upsert: + await m_store + .UpsertNodeAsync(new StoredNode(op.NodeId, op.Payload), ct) + .ConfigureAwait(false); + break; + case OutboundOpKind.Delete: + await m_store.DeleteNodeAsync(op.NodeId, ct).ConfigureAwait(false); + break; + } + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + m_logger?.LogError(ex, "Distributed address-space outbound write failed for {NodeId}.", op.NodeId); + } + } + } + catch (OperationCanceledException) + { + // shutdown + } + } + + private async Task ApplyInboundLoopAsync(ValueTask firstMove) + { + try + { + bool hasValue = await firstMove.ConfigureAwait(false); + while (hasValue) + { + NodeStateChange change = m_inboundEnumerator!.Current; + try + { + await ApplyInboundAsync(change, m_cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger?.LogError(ex, "Distributed address-space inbound apply failed for {NodeId}.", change.NodeId); + } + + InboundApplied?.Invoke(change); + hasValue = await m_inboundEnumerator.MoveNextAsync().ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // shutdown + } + } + + private async ValueTask ApplyInboundAsync(NodeStateChange change, CancellationToken cancellationToken) + { + switch (change.Kind) + { + case NodeStateChangeKind.Upsert: + if (change.Node != null) + { + await TryApplyUpsertAsync(change.NodeId, change.Node.Payload, cancellationToken) + .ConfigureAwait(false); + } + break; + case NodeStateChangeKind.Delete: + await m_addressSpace.RemoveNodeAsync(change.NodeId, cancellationToken).ConfigureAwait(false); + break; + case NodeStateChangeKind.Value: + ApplyValue(change.NodeId, change.Value); + break; + } + } + + private async ValueTask TryApplyUpsertAsync(NodeId nodeId, ByteString payload, CancellationToken cancellationToken) + { + NodeState node = NodeStateSerializer.Deserialize(m_addressSpace.Context, payload); + await m_addressSpace.AddOrUpdateNodeAsync(node, cancellationToken).ConfigureAwait(false); + } + + private void ApplyValue(NodeId nodeId, DataValue value) + { + if (m_addressSpace.TryGetNode(nodeId, out NodeState? node) && node is BaseVariableState variable) + { + variable.Value = value.WrappedValue; + variable.StatusCode = value.StatusCode; + variable.Timestamp = value.SourceTimestamp; + variable.ClearChangeMasks(m_addressSpace.Context, false); + } + } + + private void AttachStateChanged(NodeState node) + { + lock (m_lock) + { + if (m_attached.Add(node)) + { + node.StateChanged += m_onChanged; + } + } + } + + private void DetachAll() + { + lock (m_lock) + { + foreach (NodeState node in m_attached) + { + node.StateChanged -= m_onChanged; + } + m_attached.Clear(); + } + } + + private static async Task AwaitQuietlyAsync(Task? task) + { + if (task == null) + { + return; + } + try + { + await task.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + + private enum OutboundOpKind + { + Value, + Upsert, + Delete + } + + private readonly struct OutboundOp + { + private OutboundOp(OutboundOpKind kind, NodeId nodeId, DataValue value, ByteString payload) + { + Kind = kind; + NodeId = nodeId; + Value = value; + Payload = payload; + } + + public OutboundOpKind Kind { get; } + + public NodeId NodeId { get; } + + public DataValue Value { get; } + + public ByteString Payload { get; } + + public static OutboundOp ForValue(NodeId nodeId, DataValue value) + { + return new OutboundOp(OutboundOpKind.Value, nodeId, value, default); + } + + public static OutboundOp ForUpsert(NodeId nodeId, ByteString payload) + { + return new OutboundOp(OutboundOpKind.Upsert, nodeId, DataValue.Null, payload); + } + + public static OutboundOp ForDelete(NodeId nodeId) + { + return new OutboundOp(OutboundOpKind.Delete, nodeId, DataValue.Null, default); + } + } + + private readonly INodeStateStore m_store; + private readonly ILocalAddressSpace m_addressSpace; + private readonly Func m_isWriter; + private readonly ILogger? m_logger; + private readonly NodeStateChangedHandler m_onChanged; + private readonly Action m_onNodeAdded; + private readonly Action m_onNodeRemoved; + private readonly CancellationTokenSource m_cts = new(); + private readonly Lock m_lock = new(); + private readonly HashSet m_attached = []; + private Channel? m_outbound; + private IAsyncEnumerator? m_inboundEnumerator; + private Task? m_outboundTask; + private Task? m_inboundTask; + private bool m_started; + private bool m_disposed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/CrdtAddressSpaceStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/State/CrdtAddressSpaceStartupTask.cs new file mode 100644 index 0000000000..efe6a94024 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/CrdtAddressSpaceStartupTask.cs @@ -0,0 +1,134 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Crdt.Transport; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: server startup task that attaches a + /// + /// to every node manager that opts in via , + /// enabling active/active (multi-writer) replication of its address space. + /// + public sealed class CrdtAddressSpaceStartupTask : IServerStartupTask, IAsyncDisposable + { + /// + /// Creates the wiring task. + /// + /// The application service provider (for the transport factory). + /// The CRDT address-space options. + public CrdtAddressSpaceStartupTask(IServiceProvider services, ReplicatedAddressSpaceOptions options) + { + m_services = services ?? throw new ArgumentNullException(nameof(services)); + m_options = options ?? throw new ArgumentNullException(nameof(options)); + } + + /// + public async ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + ILogger logger = server.Telemetry.CreateLogger(); + + foreach (INodeManager nodeManager in server.NodeManager.NodeManagers) + { + if (nodeManager is not ILocalAddressSpaceSource source) + { + continue; + } + + ILocalAddressSpace addressSpace = source.CreateLocalAddressSpace(); + ITransport transport = m_options.CreateTransport(m_services, out InMemoryNetwork? defaultNetwork); + if (defaultNetwork != null) + { + lock (m_lock) + { + m_defaultNetworks.Add(defaultNetwork); + } + } + + var synchronizer = new CrdtAddressSpaceSynchronizer( + addressSpace, + server.MessageContext, + m_options.ReplicaId, + transport, + m_options.TimeProvider, + m_options.CreateReaderOptions(), + logger); + + await synchronizer.SeedOrHydrateAsync(cancellationToken).ConfigureAwait(false); + synchronizer.Start(); + lock (m_lock) + { + m_synchronizers.Add(synchronizer); + } + } + } + + /// + public async ValueTask DisposeAsync() + { + CrdtAddressSpaceSynchronizer[] synchronizers; + InMemoryNetwork[] networks; + lock (m_lock) + { + synchronizers = [.. m_synchronizers]; + m_synchronizers.Clear(); + networks = [.. m_defaultNetworks]; + m_defaultNetworks.Clear(); + } + + foreach (CrdtAddressSpaceSynchronizer synchronizer in synchronizers) + { + await synchronizer.DisposeAsync().ConfigureAwait(false); + } + foreach (InMemoryNetwork network in networks) + { + await network.DisposeAsync().ConfigureAwait(false); + } + } + + private readonly IServiceProvider m_services; + private readonly ReplicatedAddressSpaceOptions m_options; + private readonly Lock m_lock = new(); + private readonly List m_synchronizers = []; + private readonly List m_defaultNetworks = []; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/CrdtAddressSpaceSynchronizer.cs b/Libraries/Opc.Ua.Redundancy.Server/State/CrdtAddressSpaceSynchronizer.cs new file mode 100644 index 0000000000..290b76be04 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/CrdtAddressSpaceSynchronizer.cs @@ -0,0 +1,654 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: active/active (multi-writer) . + /// Models the node manager's address space as a last-writer-wins CRDT map + /// (keyed by node id, with separate topology and value entries) and + /// gossips full state through an . Every replica is + /// a writer: local changes mutate the local CRDT replica and broadcast, + /// while received state is merged and the resulting differences are applied + /// to the local graph. There is no leader. + /// + public sealed class CrdtAddressSpaceSynchronizer : IAddressSpaceSynchronizer + { + /// + /// Creates a CRDT address-space synchronizer. + /// + /// The local node graph adapter. + /// The message context used to encode values. + /// This replica's stable CRDT identity. + /// The gossip transport (owned by this synchronizer). + /// The time source for the logical clock. + /// Decoding limits for received state. + /// Optional logger. + public CrdtAddressSpaceSynchronizer( + ILocalAddressSpace addressSpace, + IServiceMessageContext messageContext, + ReplicaId replicaId, + ITransport transport, + TimeProvider timeProvider, + CrdtReaderOptions readerOptions, + ILogger? logger = null) + { + m_addressSpace = addressSpace ?? throw new ArgumentNullException(nameof(addressSpace)); + m_messageContext = messageContext ?? throw new ArgumentNullException(nameof(messageContext)); + m_transport = transport ?? throw new ArgumentNullException(nameof(transport)); + m_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + m_readerOptions = readerOptions ?? throw new ArgumentNullException(nameof(readerOptions)); + m_logger = logger; + m_clock = new HybridLogicalClock(replicaId, timeProvider); + m_onNodeAdded = OnLocalNodeAdded; + m_onNodeRemoved = OnLocalNodeRemoved; + m_onChanged = OnLocalNodeChanged; + m_inbound = Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + m_transport.FrameReceived += OnFrameReceived; + } + + /// + public bool IsWriter => true; + + /// + /// Raised (for tests) after each received frame has been merged and + /// applied to the local graph. + /// + internal event Action? InboundApplied; + + /// + public async ValueTask SeedOrHydrateAsync(CancellationToken ct = default) + { + await m_transport.StartAsync(ct).ConfigureAwait(false); + + byte[] snapshot; + lock (m_lock) + { + foreach (NodeState node in m_addressSpace.Nodes) + { + CaptureUpsertLocked(node); + if (node is BaseVariableState variable) + { + CaptureValueLocked(variable); + } + } + snapshot = SerializeLocked(); + } + + await m_transport.SendAsync(snapshot, ct).ConfigureAwait(false); + } + + /// + public void Start() + { + lock (m_lock) + { + if (m_started) + { + return; + } + m_started = true; + + m_addressSpace.NodeAdded += m_onNodeAdded; + m_addressSpace.NodeRemoved += m_onNodeRemoved; + foreach (NodeState node in m_addressSpace.Nodes) + { + AttachStateChanged(node); + } + } + + m_inboundTask = Task.Run(() => DrainInboundAsync(m_cts.Token)); + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_lock) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + + m_transport.FrameReceived -= OnFrameReceived; + m_addressSpace.NodeAdded -= m_onNodeAdded; + m_addressSpace.NodeRemoved -= m_onNodeRemoved; + DetachAll(); + + m_cts.Cancel(); + m_inbound.Writer.TryComplete(); + if (m_inboundTask != null) + { + try + { + await m_inboundTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + + await m_transport.DisposeAsync().ConfigureAwait(false); + m_cts.Dispose(); + } + + private void OnFrameReceived(ReadOnlyMemory frame) + { + // Copy: the transport may reuse the buffer after the callback. + m_inbound.Writer.TryWrite(frame.ToArray()); + } + + private async Task DrainInboundAsync(CancellationToken ct) + { + try + { + await foreach (byte[] frame in m_inbound.Reader.ReadAllAsync(ct).ConfigureAwait(false)) + { + try + { + await ApplyInboundFrameAsync(frame, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + m_logger?.LogError(ex, "CRDT address-space inbound apply failed."); + } + + InboundApplied?.Invoke(); + } + } + catch (OperationCanceledException) + { + // shutdown + } + } + + private async ValueTask ApplyInboundFrameAsync(byte[] frame, CancellationToken ct) + { + List diffs; + byte[] mergedSnapshot; + lock (m_lock) + { + LWWMap remote = LWWMap.ReadFrom( + frame, CrdtValues.String, ByteStringCrdtSerializer.Instance, m_readerOptions); + m_map.Merge(remote); + diffs = ComputeDiffsLocked(); + mergedSnapshot = SerializeLocked(); + } + + // Apply outside the lock; the apply path awaits the node manager and + // suppresses re-capture of its own mutations via m_applyingInbound. + bool reconciled = false; + bool previous = m_applyingInbound.Value; + m_applyingInbound.Value = true; + try + { + foreach (Diff diff in diffs) + { + await ApplyDiffAsync(diff, ct).ConfigureAwait(false); + } + + // Convergence guarantee. A value entry that won the LWW merge can fail + // to materialize on the node when a local capture and a remote value + // apply race: the capture advances m_lastApplied so ComputeDiffs no + // longer emits a correcting diff, leaving the materialized node value + // stale while the map itself is converged. A frame that produces no + // diffs is the point at which the cluster believes it is quiescent, so + // reconcile materialized node values against the converged map here. + // Re-broadcasting on a correction keeps anti-entropy alive until every + // replica's nodes match the map, after which reconciliation is a no-op + // and gossip quiesces. + if (diffs.Count == 0) + { + reconciled = ReconcileMaterializedValues(); + } + } + finally + { + m_applyingInbound.Value = previous; + } + + // Anti-entropy: when a merge changed our state (or a reconciliation + // corrected a stale materialized value), re-broadcast so the change + // propagates transitively across the cluster. This terminates once every + // replica's merge and reconciliation are no-ops (LWW is idempotent), and + // complements the transport's own gossip. + if (diffs.Count > 0 || reconciled) + { + Broadcast(mergedSnapshot); + } + } + + private bool ReconcileMaterializedValues() + { + var corrections = new List<(BaseVariableState Node, DataValue Value)>(); + lock (m_lock) + { + foreach (string key in m_map.Keys) + { + if (!key.StartsWith(ValuePrefix, StringComparison.Ordinal) || + !m_map.TryGetValue(key, out ByteString mapValue) || mapValue.IsNull || + !TryParseKey(key, out _, out NodeId nodeId) || + !m_addressSpace.TryGetNode(nodeId, out NodeState? node) || + node is not BaseVariableState variable) + { + continue; + } + + byte[] mapBytes = mapValue.ToArray(); + byte[] currentBytes = EncodeValue( + new DataValue(variable.Value, variable.StatusCode, variable.Timestamp)).ToArray(); + if (currentBytes.AsSpan().SequenceEqual(mapBytes)) + { + continue; + } + + corrections.Add((variable, DecodeValue(mapValue))); + m_lastApplied[key] = mapBytes; + } + } + + // Mutate nodes outside the lock; m_applyingInbound is set by the caller so + // the resulting ClearChangeMasks does not re-capture or re-enter the lock. + foreach ((BaseVariableState node, DataValue value) in corrections) + { + node.Value = value.WrappedValue; + node.StatusCode = value.StatusCode; + node.Timestamp = value.SourceTimestamp; + node.ClearChangeMasks(m_addressSpace.Context, false); + } + + return corrections.Count > 0; + } + + private async ValueTask ApplyDiffAsync(Diff diff, CancellationToken ct) + { + if (!TryParseKey(diff.Key, out bool isValue, out NodeId nodeId)) + { + return; + } + + if (diff.Removed) + { + if (!isValue) + { + await m_addressSpace.RemoveNodeAsync(nodeId, ct).ConfigureAwait(false); + } + return; + } + + if (isValue) + { + // Re-read the authoritative LWW value under the lock: the diff was + // computed earlier and a concurrent local write may have advanced the + // map entry since, so diff.Value can be stale. m_lastApplied is updated + // to match so ComputeDiffs stays consistent. A residual race remains + // (the node write below is outside the lock and can be overtaken by a + // concurrent local write) but is healed by ReconcileMaterializedValues + // at gossip quiescence. + ByteString authoritative; + lock (m_lock) + { + if (!m_map.TryGetValue(ValueKey(nodeId), out authoritative)) + { + return; + } + m_lastApplied[ValueKey(nodeId)] = + authoritative.IsNull ? Array.Empty() : authoritative.ToArray(); + } + if (m_addressSpace.TryGetNode(nodeId, out NodeState? node) && + node is BaseVariableState variable) + { + DataValue value = DecodeValue(authoritative); + variable.Value = value.WrappedValue; + variable.StatusCode = value.StatusCode; + variable.Timestamp = value.SourceTimestamp; + variable.ClearChangeMasks(m_addressSpace.Context, false); + } + return; + } + + NodeState reconstructed = NodeStateSerializer.Deserialize(m_addressSpace.Context, diff.Value); + + // The topology payload also carries the variable's value, but values are + // versioned independently via the value (v|) entries. Preserve the + // authoritative value entry (not a racy live-node read) so a topology merge + // never regresses or materializes a stale value that a concurrent value entry + // already advanced. + if (reconstructed is BaseVariableState reconstructedVariable) + { + ByteString authoritativeValue; + bool hasValueEntry; + lock (m_lock) + { + hasValueEntry = m_map.TryGetValue(ValueKey(nodeId), out authoritativeValue) && + !authoritativeValue.IsNull; + } + if (hasValueEntry) + { + DataValue dv = DecodeValue(authoritativeValue); + reconstructedVariable.Value = dv.WrappedValue; + reconstructedVariable.StatusCode = dv.StatusCode; + reconstructedVariable.Timestamp = dv.SourceTimestamp; + } + else if (m_addressSpace.TryGetNode(nodeId, out NodeState? existing) && + existing is BaseVariableState existingVariable) + { + reconstructedVariable.Value = existingVariable.Value; + reconstructedVariable.StatusCode = existingVariable.StatusCode; + reconstructedVariable.Timestamp = existingVariable.Timestamp; + } + } + + await m_addressSpace.AddOrUpdateNodeAsync(reconstructed, ct).ConfigureAwait(false); + AttachStateChanged(reconstructed); + } + + private void OnLocalNodeAdded(NodeState node) + { + if (m_applyingInbound.Value) + { + return; + } + + AttachStateChanged(node); + byte[] snapshot; + lock (m_lock) + { + CaptureUpsertLocked(node); + snapshot = SerializeLocked(); + } + Broadcast(snapshot); + } + + private void OnLocalNodeRemoved(NodeId nodeId) + { + if (m_applyingInbound.Value) + { + return; + } + + byte[] snapshot; + lock (m_lock) + { + m_map.Remove(TopologyKey(nodeId), m_clock); + m_map.Remove(ValueKey(nodeId), m_clock); + m_lastApplied.Remove(TopologyKey(nodeId)); + m_lastApplied.Remove(ValueKey(nodeId)); + snapshot = SerializeLocked(); + } + Broadcast(snapshot); + } + + private void OnLocalNodeChanged(ISystemContext context, NodeState node, NodeStateChangeMasks changes) + { + if (m_applyingInbound.Value || (changes & NodeStateChangeMasks.Deleted) != 0) + { + return; + } + + byte[] snapshot; + lock (m_lock) + { + if (node is BaseVariableState variable && (changes & NodeStateChangeMasks.Value) != 0) + { + CaptureValueLocked(variable); + } + + if ((changes & (NodeStateChangeMasks.NonValue | + NodeStateChangeMasks.Children | + NodeStateChangeMasks.References)) != 0) + { + CaptureUpsertLocked(node); + } + snapshot = SerializeLocked(); + } + Broadcast(snapshot); + } + + private void CaptureUpsertLocked(NodeState node) + { + string key = TopologyKey(node.NodeId); + ByteString payload = NodeStateSerializer.Serialize(m_addressSpace.Context, node); + m_map.Set(key, payload, m_clock.Now()); + UpdateLastAppliedLocked(key); + } + + private void CaptureValueLocked(BaseVariableState variable) + { + string key = ValueKey(variable.NodeId); + ByteString encoded = EncodeValue( + new DataValue(variable.Value, variable.StatusCode, variable.Timestamp)); + m_map.Set(key, encoded, m_clock.Now()); + UpdateLastAppliedLocked(key); + } + + private void UpdateLastAppliedLocked(string key) + { + if (m_map.TryGetValue(key, out ByteString stored)) + { + m_lastApplied[key] = stored.IsNull ? Array.Empty() : stored.ToArray(); + } + else + { + m_lastApplied.Remove(key); + } + } + + private byte[] SerializeLocked() + { + return m_map.ToByteArray(CrdtValues.String, ByteStringCrdtSerializer.Instance); + } + + private List ComputeDiffsLocked() + { + var diffs = new List(); + var live = new HashSet(); + + foreach (string key in m_map.Keys) + { + if (!m_map.TryGetValue(key, out ByteString current)) + { + continue; + } + live.Add(key); + byte[] currentBytes = current.ToArray(); + if (!m_lastApplied.TryGetValue(key, out byte[]? previous) || + !previous.AsSpan().SequenceEqual(currentBytes)) + { + diffs.Add(new Diff(key, current, removed: false)); + m_lastApplied[key] = currentBytes; + } + } + + foreach (string key in new List(m_lastApplied.Keys)) + { + if (!live.Contains(key)) + { + diffs.Add(new Diff(key, default, removed: true)); + m_lastApplied.Remove(key); + } + } + + return diffs; + } + + private void Broadcast(byte[] snapshot) + { + // Fire-and-forget; the gossip transport re-disseminates the latest + // frame, so a dropped send still converges. + _ = SendQuietlyAsync(snapshot); + } + + private async Task SendQuietlyAsync(byte[] snapshot) + { + try + { + await m_transport.SendAsync(snapshot, m_cts.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // shutdown + } + catch (Exception ex) + { + m_logger?.LogError(ex, "CRDT address-space broadcast failed."); + } + } + + private ByteString EncodeValue(DataValue value) + { + using var encoder = new BinaryEncoder(m_messageContext); + encoder.WriteDataValue(null, in value); + return new ByteString(encoder.CloseAndReturnBuffer()); + } + + private DataValue DecodeValue(ByteString bytes) + { + if (bytes.IsNull) + { + return DataValue.Null; + } + using var decoder = new BinaryDecoder(bytes.ToArray(), m_messageContext); + return decoder.ReadDataValue(null); + } + + private void AttachStateChanged(NodeState node) + { + lock (m_lock) + { + if (m_attached.Add(node)) + { + node.StateChanged += m_onChanged; + } + } + } + + private void DetachAll() + { + lock (m_lock) + { + foreach (NodeState node in m_attached) + { + node.StateChanged -= m_onChanged; + } + m_attached.Clear(); + } + } + + private static string TopologyKey(NodeId nodeId) + { + return TopologyPrefix + nodeId; + } + + private static string ValueKey(NodeId nodeId) + { + return ValuePrefix + nodeId; + } + + private static bool TryParseKey(string key, out bool isValue, out NodeId nodeId) + { + isValue = key.StartsWith(ValuePrefix, StringComparison.Ordinal); + bool isTopology = key.StartsWith(TopologyPrefix, StringComparison.Ordinal); + if (!isValue && !isTopology) + { + nodeId = default; + return false; + } + + try + { + nodeId = NodeId.Parse(key.Substring(2)); + return true; + } + catch (ServiceResultException) + { + nodeId = default; + return false; + } + } + + private readonly struct Diff + { + public Diff(string key, ByteString value, bool removed) + { + Key = key; + Value = value; + Removed = removed; + } + + public string Key { get; } + + public ByteString Value { get; } + + public bool Removed { get; } + } + + private const string TopologyPrefix = "n|"; + private const string ValuePrefix = "v|"; + + private readonly ILocalAddressSpace m_addressSpace; + private readonly IServiceMessageContext m_messageContext; + private readonly ITransport m_transport; + private readonly TimeProvider m_timeProvider; + private readonly CrdtReaderOptions m_readerOptions; + private readonly ILogger? m_logger; + private readonly HybridLogicalClock m_clock; + private readonly Action m_onNodeAdded; + private readonly Action m_onNodeRemoved; + private readonly NodeStateChangedHandler m_onChanged; + private readonly Channel m_inbound; + private readonly CancellationTokenSource m_cts = new(); + private readonly Lock m_lock = new(); + private readonly LWWMap m_map = new(); + private readonly Dictionary m_lastApplied = []; + private readonly HashSet m_attached = []; + private readonly AsyncLocal m_applyingInbound = new(); + private Task? m_inboundTask; + private bool m_started; + private bool m_disposed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/DictionaryAddressSpace.cs b/Libraries/Opc.Ua.Redundancy.Server/State/DictionaryAddressSpace.cs new file mode 100644 index 0000000000..d46fdf3cf3 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/DictionaryAddressSpace.cs @@ -0,0 +1,120 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: a simple dictionary-backed . Suitable + /// for hosting a flat set of top-level nodes and for unit/integration + /// testing the synchronizer; a node manager provides its own adapter + /// over PredefinedNodes in production. + /// + public sealed class DictionaryAddressSpace : ILocalAddressSpace + { + /// + /// Creates an empty address space bound to a system context. + /// + /// The system context. + public DictionaryAddressSpace(ISystemContext context) + { + Context = context ?? throw new ArgumentNullException(nameof(context)); + } + + /// + public ISystemContext Context { get; } + + /// + public IEnumerable Nodes + { + get + { + lock (m_lock) + { + return [.. m_nodes.Values]; + } + } + } + + /// + public event Action? NodeAdded; + + /// + public event Action? NodeRemoved; + + /// + public bool TryGetNode(NodeId nodeId, [NotNullWhen(true)] out NodeState? node) + { + lock (m_lock) + { + return m_nodes.TryGetValue(nodeId, out node); + } + } + + /// + public ValueTask AddOrUpdateNodeAsync(NodeState node, CancellationToken cancellationToken = default) + { + if (node == null) + { + throw new ArgumentNullException(nameof(node)); + } + + lock (m_lock) + { + m_nodes[node.NodeId] = node; + } + NodeAdded?.Invoke(node); + return default; + } + + /// + public ValueTask RemoveNodeAsync(NodeId nodeId, CancellationToken cancellationToken = default) + { + bool removed; + lock (m_lock) + { + removed = m_nodes.TryRemove(nodeId, out _); + } + if (removed) + { + NodeRemoved?.Invoke(nodeId); + } + return new ValueTask(removed); + } + + private readonly Lock m_lock = new(); + private readonly NodeIdDictionary m_nodes = []; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/DistributedAddressSpaceOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/State/DistributedAddressSpaceOptions.cs new file mode 100644 index 0000000000..b2af812671 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/DistributedAddressSpaceOptions.cs @@ -0,0 +1,117 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: options for dependency-injection registration of distributed address + /// space building blocks. + /// + public sealed class DistributedAddressSpaceOptions + { + /// + /// Gets or sets a factory that creates the record protector applied to + /// every payload written to the shared store (authenticated encryption). + /// + /// + /// When this value is null, payloads are stored without + /// encryption or integrity protection (a no-op protector). Production + /// deployments backed by a network store MUST configure an + /// ; see + /// Docs/HighAvailability.md. + /// + public Func? RecordProtectorFactory { get; set; } + + /// + /// Gets or sets a factory that creates the shared key/value store. + /// + /// + /// When this value is null, the fluent registration uses a + /// singleton . + /// + public Func? KeyValueStoreFactory { get; set; } + + /// + /// Gets or sets a value indicating whether lease-based leader election + /// should be used. + /// + /// + /// When this value is false, the fluent registration uses a + /// static single-leader election so the local server is always the + /// writer. + /// + public bool UseLeaderElection { get; set; } + + /// + /// Gets or sets the shared-store key that holds the leader lease. + /// + public string LeaseKey { get; set; } = "addressspace/leader"; + + /// + /// Gets or sets the unique identifier for this server replica. + /// + public string NodeId { get; set; } = Environment.MachineName; + + /// + /// Gets or sets how long an acquired leader lease remains valid without + /// renewal. + /// + public TimeSpan LeaseDuration { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Gets or sets how often the leader-election background loop renews + /// the lease. + /// + public TimeSpan RenewInterval { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Gets or sets the configured redundancy failover mode used for + /// service-level subrange mapping. + /// + public RedundancySupport RedundancyMode { get; set; } = RedundancySupport.Warm; + + /// + /// Gets or sets a function that returns the connected-client load used + /// to decrement healthy service levels for load balancing. + /// + public Func? ServiceLevelLoadMetric { get; set; } + + /// + /// Gets or sets a function that returns the health-derived maximum + /// service level for this replica. + /// + public Func? HealthServiceLevel { get; set; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/DistributedAddressSpaceStartupTask.cs b/Libraries/Opc.Ua.Redundancy.Server/State/DistributedAddressSpaceStartupTask.cs new file mode 100644 index 0000000000..ee3f155d31 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/DistributedAddressSpaceStartupTask.cs @@ -0,0 +1,157 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: server startup task that wires the distributed address space once the + /// server is running: it builds the with the + /// server's populated message context, registers it as the default in the + /// server's , starts leader election, + /// and attaches an to every node + /// manager that opts in via (i.e. + /// every CustomNodeManager2-derived manager). Built-in + /// infrastructure managers (Core / Diagnostics / Configuration) do not opt + /// in and are never replicated. + /// + public sealed class DistributedAddressSpaceStartupTask : IServerStartupTask, IAsyncDisposable + { + /// + /// Creates the wiring task. + /// + /// The shared key/value backend. + /// The leader election controlling writer role. + /// + /// Optional record protector applied to every payload written to the + /// shared store; defaults to a no-op pass-through. + /// + public DistributedAddressSpaceStartupTask( + ISharedKeyValueStore keyValueStore, + ILeaderElection election, + IRecordProtector? protector = null) + { + m_keyValueStore = keyValueStore ?? throw new ArgumentNullException(nameof(keyValueStore)); + m_election = election ?? throw new ArgumentNullException(nameof(election)); + m_protector = protector ?? NullRecordProtector.Instance; + } + + /// + /// The registry of instances built by + /// this task; populated once the server has started, null + /// before then. + /// + public INodeStateStoreRegistry? NodeStateStoreRegistry { get; private set; } + + /// + public async ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default) + { + if (server == null) + { + throw new ArgumentNullException(nameof(server)); + } + + ILogger logger = server.Telemetry.CreateLogger(); + + // Build the store with the server's populated message context so + // NodeId namespace indices resolve correctly. Disposal ownership is + // transferred to the registry below (NodeStateStoreRegistry.Dispose + // disposes registered stores from this task's DisposeAsync). +#pragma warning disable CA2000 + var store = new InMemoryNodeStateStore(m_keyValueStore, server.MessageContext, m_protector); +#pragma warning restore CA2000 + + // Own the node state store registry; nothing in the core server + // surface holds it. The default store is the fallback for every + // node that does not have a more specific binding. + var registry = new NodeStateStoreRegistry(server.NamespaceUris); + registry.RegisterDefault(store); + m_registry = registry; + NodeStateStoreRegistry = registry; + + // Settle leadership before seeding so the writer seeds the store and + // standbys hydrate from it, then keep renewing in the background. + await m_election.TryAcquireOrRenewAsync(cancellationToken).ConfigureAwait(false); + m_election.Start(); + + foreach (INodeManager nodeManager in server.NodeManager.NodeManagers) + { + if (nodeManager is not ILocalAddressSpaceSource source) + { + continue; + } + + ILocalAddressSpace addressSpace = source.CreateLocalAddressSpace(); + var synchronizer = new AddressSpaceSynchronizer( + store, addressSpace, () => m_election.IsLeader, logger); + await synchronizer.SeedOrHydrateAsync(cancellationToken).ConfigureAwait(false); + synchronizer.Start(); + lock (m_lock) + { + m_synchronizers.Add(synchronizer); + } + } + } + + /// + /// Stops every synchronizer and the leader election. + /// + public async ValueTask DisposeAsync() + { + AddressSpaceSynchronizer[] synchronizers; + lock (m_lock) + { + synchronizers = [.. m_synchronizers]; + m_synchronizers.Clear(); + } + + foreach (AddressSpaceSynchronizer synchronizer in synchronizers) + { + await synchronizer.DisposeAsync().ConfigureAwait(false); + } + await m_election.DisposeAsync().ConfigureAwait(false); + m_registry?.Dispose(); + } + + private readonly ISharedKeyValueStore m_keyValueStore; + private readonly ILeaderElection m_election; + private readonly IRecordProtector m_protector; + private readonly Lock m_lock = new(); + private readonly List m_synchronizers = []; + private NodeStateStoreRegistry? m_registry; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/IAddressSpaceSynchronizer.cs b/Libraries/Opc.Ua.Redundancy.Server/State/IAddressSpaceSynchronizer.cs new file mode 100644 index 0000000000..c9f8101522 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/IAddressSpaceSynchronizer.cs @@ -0,0 +1,80 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: bridges a local node graph () to a + /// shared so address-space topology and + /// variable values replicate across server replicas. + /// + /// + /// A synchronizer runs in one of two roles, selected by the + /// leader-election predicate supplied at construction: + /// + /// + /// Writer (leader): captures committed local changes and writes + /// them through to the store (shared read, master write). + /// + /// + /// Reader (standby): applies topology and value changes from the + /// store change-feed to its local graph and never writes. + /// + /// + /// Active/active multi-writer with conflict resolution is layered on top + /// later (CRDT); this single-writer model is the active/passive default. + /// + public interface IAddressSpaceSynchronizer : IAsyncDisposable + { + /// + /// true when this synchronizer currently acts as the writer + /// (leader). + /// + bool IsWriter { get; } + + /// + /// Seeds the store from the local graph when the store is empty and + /// this replica is the writer; otherwise hydrates the local graph + /// from the store. Call once before . + /// + /// Cancellation token. + ValueTask SeedOrHydrateAsync(CancellationToken ct = default); + + /// + /// Starts background replication: outbound capture for a writer, or + /// the inbound apply loop for a reader. + /// + void Start(); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateSnapshotStore.cs b/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateSnapshotStore.cs new file mode 100644 index 0000000000..5d35db9f8b --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateSnapshotStore.cs @@ -0,0 +1,131 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: optional fast-hydration capability layered on an + /// . A standby hydrates from one versioned + /// snapshot (bulk read) plus the bounded delta log of changes that occurred + /// after the snapshot, instead of streaming and applying every node one at a + /// time — cutting time-to-ready on startup and failover promotion for very + /// large address spaces. + /// + /// + /// The capability is optional: an that does not + /// also implement this interface is hydrated with the streaming + /// / + /// path. Every change and + /// snapshot entry carries a single-writer monotonic + /// so the snapshot, delta log, and + /// live feed can be applied idempotently and in order. + /// + public interface INodeStateSnapshotStore + { + /// + /// The highest write sequence assigned or observed by this store. A + /// promoted writer continues assigning sequences from this high-water + /// mark so they never move backward across a failover. + /// + ulong CurrentSequence { get; } + + /// + /// Raises the sequence high-water mark to at least + /// . Called by a standby while hydrating so a + /// later promotion keeps sequences strictly increasing. + /// + /// The observed sequence. + void ObserveSequence(ulong sequence); + + /// + /// Builds and atomically publishes a point-in-time snapshot of the + /// current node and value state, then trims the delta log up to the + /// snapshot sequence. Called by the writer (leader) off the hot path. + /// + /// Cancellation token. + ValueTask WriteSnapshotAsync(CancellationToken ct = default); + + /// + /// Reads the currently published snapshot, or null when none has + /// been published yet (in which case the caller falls back to the + /// streaming hydration path). + /// + /// Cancellation token. + /// + /// The snapshot sequence and a deferred stream of its entries, or + /// null. + /// + ValueTask TryReadSnapshotAsync(CancellationToken ct = default); + + /// + /// Streams delta-log entries with a sequence greater than + /// , in ascending sequence order, + /// so a standby can replay the changes that occurred after a snapshot. + /// + /// The exclusive lower-bound sequence. + /// Cancellation token. + IAsyncEnumerable ReadDeltaLogAsync( + ulong fromSequenceExclusive, + CancellationToken ct = default); + } + + /// + /// Extension beyond OPC 10000-4 §6.6: a published address-space snapshot — the sequence it was + /// taken at and a deferred stream of its entries (node upserts and variable + /// values, each carrying its own ). + /// + public sealed class NodeStateSnapshot + { + /// + /// Creates a snapshot handle. + /// + /// The sequence the snapshot was taken at. + /// A deferred stream of the snapshot entries. + public NodeStateSnapshot(ulong sequence, IAsyncEnumerable entries) + { + Sequence = sequence; + Entries = entries; + } + + /// + /// The write sequence the snapshot includes up to (inclusive). Delta-log + /// entries with a greater sequence must be replayed on top. + /// + public ulong Sequence { get; } + + /// + /// The snapshot entries as a deferred asynchronous stream. + /// + public IAsyncEnumerable Entries { get; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateStore.cs b/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateStore.cs new file mode 100644 index 0000000000..3a53b68c5e --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateStore.cs @@ -0,0 +1,127 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: authoritative, shareable store of a node manager's address-space + /// state — node topology and variable values — used to replicate state + /// across server replicas for high availability. + /// + /// + /// + /// The local NodeState graph remains the in-process serving + /// cache; this store is the durable/replicated copy. The + /// AddressSpaceSynchronizer bridges the two: outbound + /// write-through of committed local mutations and inbound live apply of + /// topology changes from other replicas. + /// + /// + /// The default implementation + /// () is layered on an + /// ; Redis / CRDT backends implement + /// the same contract. + /// + /// + public interface INodeStateStore + { + /// + /// Creates or replaces the stored node. + /// + /// The serialized node. + /// Cancellation token. + ValueTask UpsertNodeAsync(IStoredNode node, CancellationToken ct = default); + + /// + /// Removes the stored node. + /// + /// The node identifier. + /// Cancellation token. + /// true when a node was removed. + ValueTask DeleteNodeAsync(NodeId nodeId, CancellationToken ct = default); + + /// + /// Reads the stored node, or null when absent. + /// + /// The node identifier. + /// Cancellation token. + ValueTask TryGetNodeAsync(NodeId nodeId, CancellationToken ct = default); + + /// + /// Enumerates every stored node (used for hydration). + /// + /// Cancellation token. + IAsyncEnumerable EnumerateAsync(CancellationToken ct = default); + + /// + /// Writes the current value of a variable node. + /// + /// The variable node identifier. + /// The value to store. + /// Cancellation token. + ValueTask WriteValueAsync(NodeId nodeId, in DataValue value, CancellationToken ct = default); + + /// + /// Reads the last stored value of a variable node. + /// + /// The variable node identifier. + /// Cancellation token. + /// + /// Found = true and the stored value when present; otherwise + /// Found = false and a null . + /// + ValueTask<(bool Found, DataValue Value)> TryReadValueAsync(NodeId nodeId, CancellationToken ct = default); + + /// + /// Streams every stored variable value in a single pass (used for hydration). + /// + /// + /// A standby hydrates in two streamed passes — + /// for topology, then this for the latest values — instead of one value + /// read per node, so hydrating a large address space costs a bounded number + /// of round trips against a networked backend rather than one per variable. + /// + /// Cancellation token. + IAsyncEnumerable<(NodeId NodeId, DataValue Value)> EnumerateValuesAsync( + CancellationToken ct = default); + + /// + /// Streams topology and value changes until is + /// cancelled. Only changes that occur after the call are observed. + /// + /// Cancellation token that stops the subscription. + IAsyncEnumerable SubscribeChangesAsync(CancellationToken ct = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateStoreRegistry.cs b/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateStoreRegistry.cs new file mode 100644 index 0000000000..05eade5c90 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/INodeStateStoreRegistry.cs @@ -0,0 +1,94 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Collections.Generic; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: server-wide registry of instances, + /// resolved per node with the same three-scope precedence as the + /// historian registry: exact NodeId, then namespace, then a single + /// default fallback. + /// + public interface INodeStateStoreRegistry + { + /// + /// Adds a NodeId-scoped binding. + /// + /// The node identifier. + /// The store to bind. + void RegisterForNode(NodeId nodeId, INodeStateStore store); + + /// + /// Adds a namespace-scoped binding. + /// + /// The namespace URI. + /// The store to bind. + void RegisterForNamespace(string namespaceUri, INodeStateStore store); + + /// + /// Sets the default fallback store. + /// + /// The store to use as fallback. + void RegisterDefault(INodeStateStore store); + + /// + /// Removes the NodeId-scoped binding (no-op when absent). + /// + /// The node identifier. + /// true when a binding was removed. + bool UnregisterForNode(NodeId nodeId); + + /// + /// Removes the namespace-scoped binding (no-op when absent). + /// + /// The namespace URI. + /// true when a binding was removed. + bool UnregisterForNamespace(string namespaceUri); + + /// + /// Clears the default store. + /// + void ClearDefault(); + + /// + /// Resolves the store for a node, or null when no binding + /// matches. + /// + /// The node identifier. + INodeStateStore? Resolve(NodeId nodeId); + + /// + /// Returns a snapshot of every registered store. + /// + IReadOnlyCollection Stores { get; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/IStoredNode.cs b/Libraries/Opc.Ua.Redundancy.Server/State/IStoredNode.cs new file mode 100644 index 0000000000..0cecc85c37 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/IStoredNode.cs @@ -0,0 +1,51 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: a portable, serialized representation of a single node held in an + /// . The is produced + /// by NodeState.SaveAsBinary and consumed by + /// NodeState.LoadAsBinary, so the store never needs to reason + /// about node types (AOT-safe, reflection-free). + /// + public interface IStoredNode + { + /// + /// The identifier of the stored node. + /// + NodeId NodeId { get; } + + /// + /// The opaque, binary-encoded node payload. + /// + ByteString Payload { get; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/InMemoryNodeStateStore.cs b/Libraries/Opc.Ua.Redundancy.Server/State/InMemoryNodeStateStore.cs new file mode 100644 index 0000000000..2b74750844 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/InMemoryNodeStateStore.cs @@ -0,0 +1,859 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: default layered on an + /// . Node payloads and encoded values + /// are stored under distinct key prefixes so topology and value changes + /// can be routed independently on the change-feed. + /// + public sealed class InMemoryNodeStateStore : INodeStateStore, INodeStateSnapshotStore, IDisposable + { + /// + /// Creates a node state store over the supplied key/value backend. + /// + /// The shared key/value backend. + /// + /// The message context used to binary-encode and decode + /// payloads. + /// + /// + /// Optional record protector applied to every stored payload + /// (authenticated encryption); defaults to a no-op pass-through. + /// Configure an in production + /// so the shared store can be treated as untrusted. + /// + /// + /// The scan-poll interval used by when the backing store has no + /// change-feed (for example a CRDT gossip store). Defaults to 2 seconds. + /// + public InMemoryNodeStateStore( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector? protector = null, + TimeSpan pollInterval = default) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? NullRecordProtector.Instance; + m_pollInterval = pollInterval <= TimeSpan.Zero ? TimeSpan.FromSeconds(2) : pollInterval; + } + + /// + /// The highest write sequence this store has assigned or observed. A + /// promoted writer continues from this high-water mark so sequences + /// never move backward across a failover. + /// + public ulong CurrentSequence => unchecked((ulong)Interlocked.Read(ref m_sequence)); + + /// + /// Raises the sequence high-water mark to at least . + /// Called while hydrating (snapshot / delta log) so a later promotion + /// keeps assigning strictly increasing sequences. + /// + /// The observed sequence. + public void ObserveSequence(ulong sequence) + { + long observed = unchecked((long)sequence); + long current = Interlocked.Read(ref m_sequence); + while (current < observed) + { + long prior = Interlocked.CompareExchange(ref m_sequence, observed, current); + if (prior == current) + { + return; + } + current = prior; + } + } + + /// + /// Releases the snapshot-serialization semaphore. The backing key/value + /// store is shared and not owned by this instance. + /// + public void Dispose() + { + m_snapshotLock.Dispose(); + } + + /// + public async ValueTask UpsertNodeAsync(IStoredNode node, CancellationToken ct = default) + { + if (node == null) + { + throw new ArgumentNullException(nameof(node)); + } + ulong sequence = NextSequence(); + await m_store + .SetAsync( + NodePrefix + node.NodeId, + m_protector.Protect(WithSequence(sequence, node.Payload)), + ct) + .ConfigureAwait(false); + await AppendDeltaAsync(sequence, NodeStateChangeKind.Upsert, node.NodeId, node.Payload, ct) + .ConfigureAwait(false); + } + + /// + public async ValueTask DeleteNodeAsync(NodeId nodeId, CancellationToken ct = default) + { + ulong sequence = NextSequence(); + bool removed = await m_store.DeleteAsync(NodePrefix + nodeId, ct).ConfigureAwait(false); + await AppendDeltaAsync(sequence, NodeStateChangeKind.Delete, nodeId, ByteString.Empty, ct) + .ConfigureAwait(false); + return removed; + } + + /// + public async ValueTask TryGetNodeAsync(NodeId nodeId, CancellationToken ct = default) + { + (bool found, ByteString value) = await m_store + .TryGetAsync(NodePrefix + nodeId, ct) + .ConfigureAwait(false); + if (found && TryReadRecord(value, out _, out ByteString payload)) + { + return new StoredNode(nodeId, payload); + } + return null; + } + + /// + public async IAsyncEnumerable EnumerateAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + await foreach (KeyValuePair entry in m_store + .ScanAsync(NodePrefix, ct) + .ConfigureAwait(false)) + { + if (TryParseNodeId(entry.Key, NodePrefix, out NodeId id) && + TryReadRecord(entry.Value, out _, out ByteString payload)) + { + yield return new StoredNode(id, payload); + } + } + } + + /// + public ValueTask WriteValueAsync(NodeId nodeId, in DataValue value, CancellationToken ct = default) + { + // Encode synchronously (in-parameters are not allowed in async + // methods) and hand off to the async record + delta-log writer. + ulong sequence = NextSequence(); + ByteString payload = EncodeValue(in value); + return WriteValueRecordAsync(nodeId, sequence, payload, ct); + } + + private async ValueTask WriteValueRecordAsync( + NodeId nodeId, + ulong sequence, + ByteString payload, + CancellationToken ct) + { + await m_store + .SetAsync(ValuePrefix + nodeId, m_protector.Protect(WithSequence(sequence, payload)), ct) + .ConfigureAwait(false); + await AppendDeltaAsync(sequence, NodeStateChangeKind.Value, nodeId, payload, ct) + .ConfigureAwait(false); + } + + /// + public async ValueTask<(bool Found, DataValue Value)> TryReadValueAsync( + NodeId nodeId, + CancellationToken ct = default) + { + (bool found, ByteString value) = await m_store + .TryGetAsync(ValuePrefix + nodeId, ct) + .ConfigureAwait(false); + if (found && TryReadRecord(value, out _, out ByteString payload)) + { + return (true, DecodeValue(payload)); + } + return (false, DataValue.Null); + } + + /// + public async IAsyncEnumerable<(NodeId NodeId, DataValue Value)> EnumerateValuesAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + await foreach (KeyValuePair entry in m_store + .ScanAsync(ValuePrefix, ct) + .ConfigureAwait(false)) + { + if (TryParseNodeId(entry.Key, ValuePrefix, out NodeId id) && + TryReadRecord(entry.Value, out _, out ByteString payload)) + { + yield return (id, DecodeValue(payload)); + } + } + } + + /// + public async ValueTask WriteSnapshotAsync(CancellationToken ct = default) + { + await m_snapshotLock.WaitAsync(ct).ConfigureAwait(false); + try + { + ulong sequence = CurrentSequence; + var generation = Guid.NewGuid(); + int chunkIndex = 0; + BinaryEncoder? encoder = null; + + async ValueTask FlushAsync() + { + if (encoder == null) + { + return; + } + byte[]? plaintext = encoder.CloseAndReturnBuffer(); + encoder.Dispose(); + encoder = null; + if (plaintext == null || plaintext.Length == 0) + { + return; + } + await m_store + .SetAsync( + SnapshotChunkKey(generation, chunkIndex), + m_protector.Protect(new ByteString(plaintext)), + ct) + .ConfigureAwait(false); + chunkIndex++; + } + + void Append(byte kind, ulong entrySequence, NodeId nodeId, ByteString payload) + { + encoder ??= new BinaryEncoder(m_context); + encoder.WriteByte(null, kind); + encoder.WriteUInt64(null, entrySequence); + encoder.WriteNodeId(null, nodeId); + encoder.WriteByteString(null, payload); + } + + await foreach (KeyValuePair entry in m_store + .ScanAsync(NodePrefix, ct) + .ConfigureAwait(false)) + { + if (TryParseNodeId(entry.Key, NodePrefix, out NodeId id) && + TryReadRecord(entry.Value, out ulong seq, out ByteString payload)) + { + Append((byte)NodeStateChangeKind.Upsert, seq, id, payload); + if (encoder!.Position >= MaxChunkBytes) + { + await FlushAsync().ConfigureAwait(false); + } + } + } + + await foreach (KeyValuePair entry in m_store + .ScanAsync(ValuePrefix, ct) + .ConfigureAwait(false)) + { + if (TryParseNodeId(entry.Key, ValuePrefix, out NodeId id) && + TryReadRecord(entry.Value, out ulong seq, out ByteString payload)) + { + Append((byte)NodeStateChangeKind.Value, seq, id, payload); + if (encoder!.Position >= MaxChunkBytes) + { + await FlushAsync().ConfigureAwait(false); + } + } + } + + await FlushAsync().ConfigureAwait(false); + + // Read the manifest being replaced so its predecessor generation + // can be garbage-collected once the new manifest is published. + Guid predecessor = Guid.Empty; + Guid? generationToCollect = null; + (bool foundManifest, ByteString existingManifest) = await m_store + .TryGetAsync(ManifestKey, ct) + .ConfigureAwait(false); + if (foundManifest && TryDecodeManifest(existingManifest, out SnapshotManifest previous)) + { + predecessor = previous.Generation; + generationToCollect = previous.PreviousGeneration; + } + + // Chunks are written before the manifest, so a reader never sees a + // partial snapshot (single writer, so a plain set is sufficient). + await m_store + .SetAsync( + ManifestKey, + m_protector.Protect(EncodeManifest(generation, chunkIndex, sequence, predecessor)), + ct) + .ConfigureAwait(false); + + await TrimDeltaLogAsync(sequence, ct).ConfigureAwait(false); + + if (generationToCollect is Guid collect && collect != Guid.Empty) + { + await DeleteGenerationAsync(collect, ct).ConfigureAwait(false); + } + } + finally + { + m_snapshotLock.Release(); + } + } + + /// + public async ValueTask TryReadSnapshotAsync(CancellationToken ct = default) + { + (bool found, ByteString manifest) = await m_store + .TryGetAsync(ManifestKey, ct) + .ConfigureAwait(false); + if (!found || !TryDecodeManifest(manifest, out SnapshotManifest decoded)) + { + return null; + } + return new NodeStateSnapshot(decoded.Sequence, ReadSnapshotEntriesAsync(decoded, ct)); + } + + /// + public async IAsyncEnumerable ReadDeltaLogAsync( + ulong fromSequenceExclusive, + [EnumeratorCancellation] CancellationToken ct = default) + { + var pending = new List<(ulong Sequence, ByteString Frame)>(); + await foreach (KeyValuePair entry in m_store + .ScanAsync(DeltaPrefix, ct) + .ConfigureAwait(false)) + { + if (TryParseSequence(entry.Key, out ulong seq) && seq > fromSequenceExclusive) + { + pending.Add((seq, entry.Value)); + } + } + pending.Sort(static (left, right) => left.Sequence.CompareTo(right.Sequence)); + foreach ((ulong seq, ByteString frame) in pending) + { + NodeStateChange? change = DecodeDelta(seq, frame); + if (change != null) + { + ObserveSequence(seq); + yield return change; + } + } + } + + /// + public async IAsyncEnumerable SubscribeChangesAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + IAsyncEnumerable? feed; + try + { + feed = m_store.WatchAsync(string.Empty, ct); + } + catch (NotSupportedException) + { + // The backing store has no change-feed (for example a CRDT + // gossip store, or the bulk side of a hybrid store). Fall back + // to periodic scan-polling so a standby replica still tracks + // topology/value changes instead of silently stopping. + feed = null; + } + + if (feed != null) + { + await foreach (KeyValueChange change in feed.ConfigureAwait(false)) + { + NodeStateChange? mapped = Map(change); + if (mapped != null) + { + yield return mapped; + } + } + yield break; + } + + await foreach (NodeStateChange change in PollChangesAsync(ct).ConfigureAwait(false)) + { + yield return change; + } + } + + private async IAsyncEnumerable PollChangesAsync( + [EnumeratorCancellation] CancellationToken ct) + { + var nodes = new Dictionary(StringComparer.Ordinal); + var values = new Dictionary(StringComparer.Ordinal); + bool baseline = false; + + while (!ct.IsCancellationRequested) + { + Dictionary currentNodes = + await SnapshotPrefixAsync(NodePrefix, ct).ConfigureAwait(false); + Dictionary currentValues = + await SnapshotPrefixAsync(ValuePrefix, ct).ConfigureAwait(false); + + if (baseline) + { + // Only changes after the first (baseline) scan are emitted, + // matching the change-feed "observe changes after the call" + // contract; the synchronizer hydrates the initial snapshot + // separately via EnumerateAsync. + foreach (NodeStateChange change in DiffSet(nodes, currentNodes)) + { + yield return change; + } + foreach (NodeStateChange change in DiffDeletes(nodes, currentNodes)) + { + yield return change; + } + foreach (NodeStateChange change in DiffSet(values, currentValues)) + { + yield return change; + } + } + + nodes = currentNodes; + values = currentValues; + baseline = true; + await Task.Delay(m_pollInterval, ct).ConfigureAwait(false); + } + } + + private IEnumerable DiffSet( + Dictionary previous, + Dictionary current) + { + foreach (KeyValuePair entry in current) + { + if (!previous.TryGetValue(entry.Key, out ByteString prior) || !prior.Equals(entry.Value)) + { + NodeStateChange? mapped = Map(new KeyValueChange + { + Kind = KeyValueChangeKind.Set, + Key = entry.Key, + Value = entry.Value + }); + if (mapped != null) + { + yield return mapped; + } + } + } + } + + private IEnumerable DiffDeletes( + Dictionary previous, + Dictionary current) + { + foreach (string key in previous.Keys) + { + if (!current.ContainsKey(key)) + { + NodeStateChange? mapped = Map(new KeyValueChange + { + Kind = KeyValueChangeKind.Delete, + Key = key + }); + if (mapped != null) + { + yield return mapped; + } + } + } + } + + private async Task> SnapshotPrefixAsync(string prefix, CancellationToken ct) + { + var snapshot = new Dictionary(StringComparer.Ordinal); + await foreach (KeyValuePair entry in m_store + .ScanAsync(prefix, ct) + .ConfigureAwait(false)) + { + snapshot[entry.Key] = entry.Value; + } + return snapshot; + } + + private NodeStateChange? Map(KeyValueChange change) + { + if (change.Key.StartsWith(NodePrefix, StringComparison.Ordinal)) + { + if (!TryParseNodeId(change.Key, NodePrefix, out NodeId id)) + { + return null; + } + if (change.Kind == KeyValueChangeKind.Delete) + { + return new NodeStateChange { Kind = NodeStateChangeKind.Delete, NodeId = id }; + } + if (!TryReadRecord(change.Value, out ulong sequence, out ByteString payload)) + { + return null; + } + return new NodeStateChange + { + Kind = NodeStateChangeKind.Upsert, + NodeId = id, + Node = new StoredNode(id, payload), + Sequence = sequence + }; + } + + if (change.Key.StartsWith(ValuePrefix, StringComparison.Ordinal)) + { + if (change.Kind != KeyValueChangeKind.Set || + !TryParseNodeId(change.Key, ValuePrefix, out NodeId id)) + { + return null; + } + if (!TryReadRecord(change.Value, out ulong sequence, out ByteString payload)) + { + return null; + } + return new NodeStateChange + { + Kind = NodeStateChangeKind.Value, + NodeId = id, + Value = DecodeValue(payload), + Sequence = sequence + }; + } + + return null; + } + + private ByteString EncodeValue(in DataValue value) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteDataValue(null, in value); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : new ByteString(buffer); + } + + private DataValue DecodeValue(ByteString bytes) + { + if (bytes.IsNull || bytes.IsEmpty) + { + return DataValue.Null; + } + using var decoder = new BinaryDecoder(bytes.ToArray(), m_context); + return decoder.ReadDataValue(null); + } + + private ValueTask AppendDeltaAsync( + ulong sequence, + NodeStateChangeKind kind, + NodeId nodeId, + ByteString payload, + CancellationToken ct) + { + return m_store.SetAsync( + DeltaPrefix + FormatSequence(sequence), + m_protector.Protect(EncodeDelta(kind, nodeId, payload)), + ct); + } + + private ByteString EncodeDelta(NodeStateChangeKind kind, NodeId nodeId, ByteString payload) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteByte(null, (byte)kind); + encoder.WriteNodeId(null, nodeId); + encoder.WriteByteString(null, payload); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : new ByteString(buffer); + } + + private NodeStateChange? DecodeDelta(ulong sequence, ByteString frame) + { + if (!m_protector.TryUnprotect(frame, out ByteString plaintext) || plaintext.IsNull) + { + return null; + } + using var decoder = new BinaryDecoder(plaintext.ToArray(), m_context); + var kind = (NodeStateChangeKind)decoder.ReadByte(null); + NodeId nodeId = decoder.ReadNodeId(null); + ByteString payload = decoder.ReadByteString(null); + return kind switch + { + NodeStateChangeKind.Upsert => new NodeStateChange + { + Kind = NodeStateChangeKind.Upsert, + NodeId = nodeId, + Node = new StoredNode(nodeId, payload), + Sequence = sequence + }, + NodeStateChangeKind.Value => new NodeStateChange + { + Kind = NodeStateChangeKind.Value, + NodeId = nodeId, + Value = DecodeValue(payload), + Sequence = sequence + }, + NodeStateChangeKind.Delete => new NodeStateChange + { + Kind = NodeStateChangeKind.Delete, + NodeId = nodeId, + Sequence = sequence + }, + _ => null + }; + } + + private async IAsyncEnumerable ReadSnapshotEntriesAsync( + SnapshotManifest manifest, + [EnumeratorCancellation] CancellationToken ct) + { + for (int i = 0; i < manifest.ChunkCount; i++) + { + (bool found, ByteString chunk) = await m_store + .TryGetAsync(SnapshotChunkKey(manifest.Generation, i), ct) + .ConfigureAwait(false); + if (!found || !m_protector.TryUnprotect(chunk, out ByteString plaintext) || plaintext.IsNull) + { + continue; + } + + byte[] buffer = plaintext.ToArray(); + using var decoder = new BinaryDecoder(buffer, m_context); + while (decoder.Position < buffer.Length) + { + var kind = (NodeStateChangeKind)decoder.ReadByte(null); + ulong entrySequence = decoder.ReadUInt64(null); + NodeId nodeId = decoder.ReadNodeId(null); + ByteString payload = decoder.ReadByteString(null); + NodeStateChange? change = kind switch + { + NodeStateChangeKind.Upsert => new NodeStateChange + { + Kind = NodeStateChangeKind.Upsert, + NodeId = nodeId, + Node = new StoredNode(nodeId, payload), + Sequence = entrySequence + }, + NodeStateChangeKind.Value => new NodeStateChange + { + Kind = NodeStateChangeKind.Value, + NodeId = nodeId, + Value = DecodeValue(payload), + Sequence = entrySequence + }, + _ => null + }; + if (change != null) + { + yield return change; + } + } + } + } + + private async ValueTask TrimDeltaLogAsync(ulong throughSequenceInclusive, CancellationToken ct) + { + var stale = new List(); + await foreach (KeyValuePair entry in m_store + .ScanAsync(DeltaPrefix, ct) + .ConfigureAwait(false)) + { + if (TryParseSequence(entry.Key, out ulong seq) && seq <= throughSequenceInclusive) + { + stale.Add(entry.Key); + } + } + foreach (string key in stale) + { + await m_store.DeleteAsync(key, ct).ConfigureAwait(false); + } + } + + private async ValueTask DeleteGenerationAsync(Guid generation, CancellationToken ct) + { + var keys = new List(); + await foreach (KeyValuePair entry in m_store + .ScanAsync(SnapshotChunkPrefix(generation), ct) + .ConfigureAwait(false)) + { + keys.Add(entry.Key); + } + foreach (string key in keys) + { + await m_store.DeleteAsync(key, ct).ConfigureAwait(false); + } + } + + private ByteString EncodeManifest(Guid generation, int chunkCount, ulong sequence, Guid previousGeneration) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteByteString(null, new ByteString(generation.ToByteArray())); + encoder.WriteInt32(null, chunkCount); + encoder.WriteUInt64(null, sequence); + encoder.WriteByteString(null, new ByteString(previousGeneration.ToByteArray())); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : new ByteString(buffer); + } + + private bool TryDecodeManifest(ByteString stored, out SnapshotManifest manifest) + { + manifest = default; + if (!m_protector.TryUnprotect(stored, out ByteString plaintext) || plaintext.IsNull) + { + return false; + } + try + { + using var decoder = new BinaryDecoder(plaintext.ToArray(), m_context); + var generation = new Guid(decoder.ReadByteString(null).ToArray()); + int chunkCount = decoder.ReadInt32(null); + ulong sequence = decoder.ReadUInt64(null); + var previous = new Guid(decoder.ReadByteString(null).ToArray()); + manifest = new SnapshotManifest(generation, chunkCount, sequence, previous); + return true; + } + catch (ServiceResultException) + { + return false; + } + } + + private static string SnapshotChunkPrefix(Guid generation) + { + return SnapshotPrefix + generation.ToString("N") + "/"; + } + + private static string SnapshotChunkKey(Guid generation, int chunkIndex) + { + return SnapshotChunkPrefix(generation) + chunkIndex.ToString("D8", CultureInfo.InvariantCulture); + } + + private static string FormatSequence(ulong sequence) + { + return sequence.ToString("D20", CultureInfo.InvariantCulture); + } + + private static bool TryParseSequence(string key, out ulong sequence) + { + sequence = 0; + int slash = key.LastIndexOf('/'); + return slash >= 0 && + ulong.TryParse( + key.Substring(slash + 1), + NumberStyles.None, + CultureInfo.InvariantCulture, + out sequence); + } + + private static bool TryParseNodeId(string key, string prefix, out NodeId nodeId) + { + nodeId = NodeId.Null; + if (key.Length <= prefix.Length) + { + return false; + } + try + { + nodeId = NodeId.Parse(key.Substring(prefix.Length)); + return !nodeId.IsNull; + } + catch (ServiceResultException) + { + return false; + } + } + + private ulong NextSequence() + { + return unchecked((ulong)Interlocked.Increment(ref m_sequence)); + } + + private bool TryReadRecord(ByteString stored, out ulong sequence, out ByteString payload) + { + sequence = 0; + payload = ByteString.Empty; + return m_protector.TryUnprotect(stored, out ByteString wrapped) && + TrySplitSequence(wrapped, out sequence, out payload); + } + + private static ByteString WithSequence(ulong sequence, ByteString payload) + { + int length = payload.IsNull ? 0 : payload.Length; + byte[] buffer = new byte[SequencePrefixLength + length]; + BinaryPrimitives.WriteUInt64BigEndian(buffer.AsSpan(0, SequencePrefixLength), sequence); + if (length > 0) + { + payload.Span.CopyTo(buffer.AsSpan(SequencePrefixLength)); + } + return new ByteString(buffer); + } + + private static bool TrySplitSequence(ByteString wrapped, out ulong sequence, out ByteString payload) + { + sequence = 0; + payload = ByteString.Empty; + if (wrapped.IsNull || wrapped.Length < SequencePrefixLength) + { + return false; + } + ReadOnlySpan span = wrapped.Span; + sequence = BinaryPrimitives.ReadUInt64BigEndian(span.Slice(0, SequencePrefixLength)); + payload = new ByteString(span.Slice(SequencePrefixLength).ToArray()); + return true; + } + + private const int SequencePrefixLength = 8; + private const int MaxChunkBytes = 1024 * 1024; + private const string NodePrefix = "n/"; + private const string ValuePrefix = "v/"; + private const string DeltaPrefix = "dlog/"; + private const string SnapshotPrefix = "snap/"; + private const string ManifestKey = "snapmeta/manifest"; + private long m_sequence; + private readonly SemaphoreSlim m_snapshotLock = new(1, 1); + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + private readonly TimeSpan m_pollInterval; + + /// + /// The published-snapshot pointer: which generation of chunks is live, + /// how many chunks it has, the sequence it includes up to, and the + /// predecessor generation retained for readers still draining it. + /// + private readonly record struct SnapshotManifest( + Guid Generation, + int ChunkCount, + ulong Sequence, + Guid PreviousGeneration); + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateChange.cs b/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateChange.cs new file mode 100644 index 0000000000..6185903047 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateChange.cs @@ -0,0 +1,94 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: the kind of node-state change reported by an + /// change-feed. + /// + public enum NodeStateChangeKind + { + /// + /// A node was created or its topology/attributes were updated. + /// + Upsert, + + /// + /// A node was removed. + /// + Delete, + + /// + /// A variable value (Value/StatusCode/Timestamp) changed. + /// + Value + } + + /// + /// Extension beyond OPC 10000-4 §6.6: a single change observed on an + /// change-feed. Topology changes ( + /// / ) are applied live to every + /// replica; changes carry the + /// new . + /// + public sealed record NodeStateChange + { + /// + /// The kind of change. + /// + public NodeStateChangeKind Kind { get; init; } + + /// + /// The affected node identifier. + /// + public NodeId NodeId { get; init; } = NodeId.Null; + + /// + /// The serialized node for + /// changes; otherwise null. + /// + public IStoredNode? Node { get; init; } + + /// + /// The new value for changes; + /// otherwise a null + /// (). + /// + public DataValue Value { get; init; } = DataValue.Null; + + /// + /// The monotonic write sequence of this change, assigned by the single + /// writer. Used to order and de-duplicate changes across the snapshot, + /// delta-log, and live-feed apply paths. 0 when the backing store + /// does not track sequences. + /// + public ulong Sequence { get; init; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateSerializer.cs b/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateSerializer.cs new file mode 100644 index 0000000000..cb80d314b9 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateSerializer.cs @@ -0,0 +1,126 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Buffers.Binary; +using System.IO; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: serializes a to a portable, self-describing + /// payload and reconstructs it. The payload is framed as a 4-byte + /// little-endian followed by the standard + /// NodeState.SaveAsBinary encoding, so a replica can reconstruct + /// a generic node of the correct class without knowing the original + /// concrete (possibly source-generated) type. + /// + /// + /// Reconstruction yields the matching generic base state + /// (, , + /// …). Type-specific behavior (method handlers, custom callbacks) is not + /// carried in the payload — it is re-attached by the owning node manager + /// on the active replica. This is sufficient for browse / read / value + /// replication and active/passive failover. + /// + public static class NodeStateSerializer + { + /// + /// Serializes a node (and its children/references) to a framed + /// binary payload. + /// + /// The system context for encoding. + /// The node to serialize. + public static ByteString Serialize(ISystemContext context, NodeState node) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + if (node == null) + { + throw new ArgumentNullException(nameof(node)); + } + + using var stream = new MemoryStream(); + byte[] header = new byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(header, (int)node.NodeClass); + stream.Write(header, 0, 4); + node.SaveAsBinary(context, stream); + return new ByteString(stream.ToArray()); + } + + /// + /// Reconstructs a node from a payload produced by + /// . + /// + /// The system context for decoding. + /// The framed binary payload. + public static NodeState Deserialize(ISystemContext context, ByteString payload) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + byte[] bytes = payload.ToArray(); + if (bytes.Length < 4) + { + throw new ServiceResultException( + StatusCodes.BadDecodingError, + "Distributed node payload is too short to contain a node class header."); + } + + var nodeClass = (NodeClass)BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(0, 4)); + NodeState node = Create(nodeClass); + using var stream = new MemoryStream(bytes, 4, bytes.Length - 4, writable: false); + node.LoadAsBinary(context, stream); + return node; + } + + private static NodeState Create(NodeClass nodeClass) + { + return nodeClass switch + { + NodeClass.Object => new BaseObjectState(null), + NodeClass.Variable => new BaseDataVariableState(null), + NodeClass.Method => new MethodState(null), + NodeClass.View => new ViewState(), + NodeClass.ObjectType => new BaseObjectTypeState(), + NodeClass.VariableType => new BaseDataVariableTypeState(), + NodeClass.ReferenceType => new ReferenceTypeState(), + NodeClass.DataType => new DataTypeState(), + _ => throw new ServiceResultException( + StatusCodes.BadNodeClassInvalid, + $"Cannot reconstruct a node of class {nodeClass}.") + }; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateStoreRegistry.cs b/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateStoreRegistry.cs new file mode 100644 index 0000000000..e646ba3d28 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/NodeStateStoreRegistry.cs @@ -0,0 +1,265 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: default implementation — a + /// thread-safe lookup keyed by NodeId, namespace URI and a single + /// default fallback. + /// + public sealed class NodeStateStoreRegistry : INodeStateStoreRegistry, IDisposable + { + /// + /// Creates an empty registry bound to a namespace table. + /// + /// + /// The namespace table used to resolve namespace-scoped bindings. + /// + public NodeStateStoreRegistry(NamespaceTable namespaceTable) + { + m_namespaceTable = namespaceTable ?? throw new ArgumentNullException(nameof(namespaceTable)); + } + + /// + public IReadOnlyCollection Stores + { + get + { + lock (m_lock) + { + return [.. m_stores]; + } + } + } + + /// + public void RegisterForNode(NodeId nodeId, INodeStateStore store) + { + if (nodeId.IsNull) + { + throw new ArgumentException("NodeId must not be null.", nameof(nodeId)); + } + if (store == null) + { + throw new ArgumentNullException(nameof(store)); + } + + lock (m_lock) + { + m_nodes[nodeId] = store; + m_stores.Add(store); + } + } + + /// + public void RegisterForNamespace(string namespaceUri, INodeStateStore store) + { + if (string.IsNullOrEmpty(namespaceUri)) + { + throw new ArgumentException("Namespace URI must not be null or empty.", nameof(namespaceUri)); + } + if (store == null) + { + throw new ArgumentNullException(nameof(store)); + } + + lock (m_lock) + { + m_namespaces[namespaceUri] = store; + m_stores.Add(store); + } + } + + /// + public void RegisterDefault(INodeStateStore store) + { + if (store == null) + { + throw new ArgumentNullException(nameof(store)); + } + + lock (m_lock) + { + m_default = store; + m_stores.Add(store); + } + } + + /// + public bool UnregisterForNode(NodeId nodeId) + { + if (nodeId.IsNull) + { + return false; + } + + lock (m_lock) + { + if (m_nodes.TryGetValue(nodeId, out INodeStateStore? store)) + { + m_nodes.Remove(nodeId); + RebuildStoreSet(store); + return true; + } + return false; + } + } + + /// + public bool UnregisterForNamespace(string namespaceUri) + { + if (string.IsNullOrEmpty(namespaceUri)) + { + return false; + } + + lock (m_lock) + { + if (m_namespaces.TryGetValue(namespaceUri, out INodeStateStore? store)) + { + m_namespaces.Remove(namespaceUri); + RebuildStoreSet(store); + return true; + } + return false; + } + } + + /// + public void ClearDefault() + { + lock (m_lock) + { + INodeStateStore? prev = m_default; + m_default = null; + if (prev != null) + { + RebuildStoreSet(prev); + } + } + } + + /// + public INodeStateStore? Resolve(NodeId nodeId) + { + if (nodeId.IsNull) + { + return null; + } + + lock (m_lock) + { + if (m_nodes.TryGetValue(nodeId, out INodeStateStore? byNode)) + { + return byNode; + } + + if (m_namespaces.Count > 0) + { + string? uri = m_namespaceTable.GetString(nodeId.NamespaceIndex); + if (uri != null && m_namespaces.TryGetValue(uri, out INodeStateStore? byNamespace)) + { + return byNamespace; + } + } + + return m_default; + } + } + + /// + /// Disposes any registered store that implements + /// . + /// + public void Dispose() + { + HashSet stores; + lock (m_lock) + { + stores = [.. m_stores]; + m_nodes.Clear(); + m_namespaces.Clear(); + m_default = null; + m_stores.Clear(); + } + + foreach (INodeStateStore store in stores) + { + if (store is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + + private void RebuildStoreSet(INodeStateStore candidate) + { + if (!ContainsStore(candidate)) + { + m_stores.Remove(candidate); + } + } + + private bool ContainsStore(INodeStateStore candidate) + { + if (ReferenceEquals(candidate, m_default)) + { + return true; + } + foreach (INodeStateStore value in m_nodes.Values) + { + if (ReferenceEquals(value, candidate)) + { + return true; + } + } + foreach (INodeStateStore value in m_namespaces.Values) + { + if (ReferenceEquals(value, candidate)) + { + return true; + } + } + return false; + } + + private readonly Lock m_lock = new(); + private readonly NamespaceTable m_namespaceTable; + private readonly NodeIdDictionary m_nodes = []; + private readonly Dictionary m_namespaces = new(StringComparer.Ordinal); + private readonly HashSet m_stores = []; + private INodeStateStore? m_default; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/ReplicatedAddressSpaceOptions.cs b/Libraries/Opc.Ua.Redundancy.Server/State/ReplicatedAddressSpaceOptions.cs new file mode 100644 index 0000000000..ecae89904d --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/ReplicatedAddressSpaceOptions.cs @@ -0,0 +1,40 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: options for CRDT active/active address-space replication. Topology and + /// variable values are modelled as a last-writer-wins map and gossiped + /// between replicas; every replica is a writer. + /// + public sealed class ReplicatedAddressSpaceOptions : ReplicatedGossipOptions + { + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/State/StoredNode.cs b/Libraries/Opc.Ua.Redundancy.Server/State/StoredNode.cs new file mode 100644 index 0000000000..9623150668 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/State/StoredNode.cs @@ -0,0 +1,54 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: default immutable implementation. + /// + public sealed class StoredNode : IStoredNode + { + /// + /// Creates a stored node from an identifier and a binary payload. + /// + /// The node identifier. + /// The binary-encoded node payload. + public StoredNode(NodeId nodeId, ByteString payload) + { + NodeId = nodeId; + Payload = payload; + } + + /// + public NodeId NodeId { get; } + + /// + public ByteString Payload { get; } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/DeterministicEventIdProvider.cs b/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/DeterministicEventIdProvider.cs new file mode 100644 index 0000000000..48e15b6e20 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/DeterministicEventIdProvider.cs @@ -0,0 +1,114 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using Opc.Ua.Server.Fluent; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Optional EventId synchronization provider for Transparent and HotAndMirrored redundancy. + /// + /// + /// OPC 10000-4 §6.6.2.2 requires EventIds to be synchronized for Transparent and HotAndMirrored + /// RedundantServerSets so clients do not double-process events after Failover. This provider is + /// deterministic for the same replica-set seed, notifier, event type, source, source timestamp, severity, and + /// message. It intentionally excludes per-replica ReceiveTime. Events that lack stable distinguishing + /// fields should set EventId explicitly or use a stronger application-level event identity. + /// + public sealed class DeterministicEventIdProvider : IEventIdProvider + { + /// + /// Creates a deterministic event id provider. + /// + /// + /// A stable, non-secret seed shared by all replicas in the transparent set. + /// + /// is empty. + public DeterministicEventIdProvider(string replicaSetSeed) + { + if (string.IsNullOrWhiteSpace(replicaSetSeed)) + { + throw new ArgumentException("A replica-set seed is required.", nameof(replicaSetSeed)); + } + + m_replicaSetSeed = replicaSetSeed; + } + + /// + public ByteString CreateEventId(BaseObjectState notifier, ISystemContext context, BaseEventState eventState) + { + if (notifier == null) + { + throw new ArgumentNullException(nameof(notifier)); + } + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + if (eventState == null) + { + throw new ArgumentNullException(nameof(eventState)); + } + + var builder = new StringBuilder(); + Append(builder, m_replicaSetSeed); + Append(builder, notifier.NodeId.ToString()); + Append(builder, (eventState.EventType?.Value ?? eventState.GetDefaultTypeDefinitionId(context)).ToString()); + Append(builder, (eventState.SourceNode?.Value ?? notifier.NodeId).ToString()); + Append(builder, eventState.Time?.Value.ToString(CultureInfo.InvariantCulture) ?? string.Empty); + Append(builder, eventState.Severity?.Value.ToString(CultureInfo.InvariantCulture) ?? string.Empty); + Append(builder, eventState.Message?.Value.Text ?? string.Empty); + + byte[] bytes = Encoding.UTF8.GetBytes(builder.ToString()); +#if NET8_0_OR_GREATER + return ByteString.From(SHA256.HashData(bytes)); +#else + using SHA256 sha = SHA256.Create(); + return ByteString.From(sha.ComputeHash(bytes)); +#endif + } + + private static void Append(StringBuilder builder, string value) + { + builder.Append(value.Length.ToString(CultureInfo.InvariantCulture)); + builder.Append(':'); + builder.Append(value); + builder.Append('|'); + } + + private readonly string m_replicaSetSeed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/DistributedSubscriptionBuilderExtensions.cs b/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/DistributedSubscriptionBuilderExtensions.cs new file mode 100644 index 0000000000..64e7d1345d --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/DistributedSubscriptionBuilderExtensions.cs @@ -0,0 +1,78 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Fluent registration for OPC 10000-4 §6.6.2.2 cross-replica Subscription definition mirroring. + /// + public static class DistributedSubscriptionBuilderExtensions + { + /// + /// Registers the shared-store backed Subscription definition mirror. + /// + /// + /// The registered persists subscription and monitored-item definitions, + /// retransmission state for Republish, and continuation-point envelopes. This provides the + /// HighAvailability capability boundary for continuation points: transferred sessions can identify and release + /// mirrored opaque continuation tokens, but the local Browse/Query/History continuation enumerator state is not + /// rebuilt on another replica. Monitored-item data/event queues remain outside this registration's scope. + /// + /// The server builder. + /// The same for chaining. + /// is null. + public static IOpcUaServerBuilder UseDistributedSubscriptionMirroring(this IOpcUaServerBuilder builder) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + builder.Services.TryAddSingleton(_ => new InMemorySharedKeyValueStore()); + builder.Services.TryAddSingleton(sp => + new SharedKeyValueSubscriptionStore( + sp.GetRequiredService(), + sp.GetRequiredService(), + RecordProtectionGuard.ResolveProtectorOrThrow(sp), + sp.GetService>())); + + return builder; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/SharedKeyValueSubscriptionStore.cs b/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/SharedKeyValueSubscriptionStore.cs new file mode 100644 index 0000000000..e2d0fb0fc1 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Subscriptions/SharedKeyValueSubscriptionStore.cs @@ -0,0 +1,1156 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Redundancy; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Shared key/value backed for HotAndMirrored/Transparent subscription state + /// synchronization. + /// + /// + /// This store supports OPC 10000-4 §6.6.2.4.4 and §6.6.2.4.5.5 by mirroring subscription definitions, + /// retransmission state for Republish, and continuation-point envelopes (§6.6.2.2). Continuation + /// mirroring is envelope-only: the opaque continuation token, owner SessionId, kind, and expiry metadata are + /// persisted so a backup can reject, release, or correlate the token, but Browse/Query/History continuation + /// enumerator internals owned by a local node manager remain process-local runtime state and are not resumed by + /// this store. + /// Monitored-item data/event queues remain runtime state and are not restored by this store. + /// + public sealed class SharedKeyValueSubscriptionStore : + ISubscriptionStore, + ISubscriptionRetransmissionDeltaStore, + IContinuationPointStore, + IAsyncDisposable + { + /// + /// Creates a subscription definition store over a shared key/value backend. + /// + /// The shared key/value backend. + /// The message context for encoding. + /// + /// Optional record protector applied to every encoded subscription entry; defaults to pass-through. + /// + /// Optional logger for asynchronous mirror failures. + public SharedKeyValueSubscriptionStore( + ISharedKeyValueStore store, + IServiceMessageContext context, + IRecordProtector? protector = null, + ILogger? logger = null) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_protector = protector ?? NullRecordProtector.Instance; + m_logger = logger; + m_definitionCache = s_definitionCaches.GetValue(store, static _ => new SharedDefinitionCache()); + m_channel = Channel.CreateBounded(new BoundedChannelOptions(ChannelCapacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = false + }); + m_drainTask = Task.Run(DrainAsync); + } + + /// + public async ValueTask StoreSubscriptionsAsync( + IEnumerable subscriptions, + CancellationToken cancellationToken = default) + { + if (subscriptions == null) + { + throw new ArgumentNullException(nameof(subscriptions)); + } + + List snapshot = subscriptions.Select(CloneSubscription).ToList(); + var liveIds = new HashSet(); + foreach (StoredSubscription subscription in snapshot) + { + liveIds.Add(subscription.Id); + string key = KeyFor(subscription.Id); + await m_store + .SetAsync(key, m_protector.Protect(Encode(subscription)), cancellationToken) + .ConfigureAwait(false); + } + + uint[] removedIds; + lock (m_definitionCache.Lock) + { + removedIds = m_definitionCache.Subscriptions.Keys + .Where(id => !liveIds.Contains(id)) + .ToArray(); + foreach (StoredSubscription subscription in snapshot) + { + m_definitionCache.Subscriptions[subscription.Id] = CloneSubscription(subscription); + } + foreach (uint subscriptionId in removedIds) + { + m_definitionCache.Subscriptions.Remove(subscriptionId); + } + } + + foreach (uint subscriptionId in removedIds) + { + await m_store.DeleteAsync(KeyFor(subscriptionId), cancellationToken).ConfigureAwait(false); + DeleteRetransmissionState(subscriptionId); + } + + return true; + } + + /// + public ValueTask RestoreSubscriptionsAsync( + CancellationToken cancellationToken = default) + { + lock (m_definitionCache.Lock) + { + return new ValueTask(new RestoreSubscriptionResult( + true, + m_definitionCache.Subscriptions.Values.Select(CloneSubscription).ToList())); + } + } + + /// + public IDataChangeMonitoredItemQueue RestoreDataChangeMonitoredItemQueue(uint monitoredItemId) + { + return null!; + } + + /// + public IEventMonitoredItemQueue RestoreEventMonitoredItemQueue(uint monitoredItemId) + { + return null!; + } + + /// + public async ValueTask OnSubscriptionRestoreCompleteAsync( + Dictionary> createdSubscriptions, + CancellationToken cancellationToken = default) + { + if (createdSubscriptions == null) + { + throw new ArgumentNullException(nameof(createdSubscriptions)); + } + + uint[] removedIds; + lock (m_definitionCache.Lock) + { + var liveIds = new HashSet(createdSubscriptions.Keys); + removedIds = m_definitionCache.Subscriptions.Keys + .Where(id => !liveIds.Contains(id)) + .ToArray(); + foreach (uint subscriptionId in removedIds) + { + m_definitionCache.Subscriptions.Remove(subscriptionId); + } + } + + foreach (uint subscriptionId in removedIds) + { + await m_store.DeleteAsync(KeyFor(subscriptionId), cancellationToken).ConfigureAwait(false); + DeleteRetransmissionState(subscriptionId); + } + } + + /// + public async ValueTask LoadRetransmissionStateAsync( + uint subscriptionId, + CancellationToken cancellationToken = default) + { + (bool found, ByteString value) = await m_store + .TryGetAsync(RetransmissionStateKeyFor(subscriptionId), cancellationToken) + .ConfigureAwait(false); + if (!found || !m_protector.TryUnprotect(value, out ByteString payload)) + { + return null; + } + + using var decoder = new BinaryDecoder(payload.ToArray(), m_context); + int version = decoder.ReadInt32(null); + if (version is not RetransmissionStateFormatVersion and not LegacyRetransmissionStateFormatVersion) + { + return null; + } + + var state = new SubscriptionRetransmissionState + { + NextSequenceNumber = decoder.ReadUInt32(null) + }; + NamespaceTable? namespaceUris = null; + StringTable? serverUris = null; + if (version == RetransmissionStateFormatVersion) + { + namespaceUris = CreateNamespaceTable(decoder.ReadStringArray(null)); + serverUris = CreateStringTable(decoder.ReadStringArray(null)); + } + + var messages = new List(); + await foreach (KeyValuePair pair in m_store + .ScanAsync(RetransmissionMessagePrefixFor(subscriptionId), cancellationToken) + .ConfigureAwait(false)) + { + if (m_protector.TryUnprotect(pair.Value, out ByteString messagePayload)) + { + NotificationMessage message = DecodeNotificationMessage(messagePayload, namespaceUris, serverUris); + messages.Add(message); + } + } + messages.Sort(static (left, right) => left.SequenceNumber.CompareTo(right.SequenceNumber)); + state.SentMessages = [.. messages]; + return state; + } + + /// + public void StoreRetransmissionState( + uint subscriptionId, + uint nextSequenceNumber, + ArrayOf sentMessages) + { + lock (m_retransmissionLock) + { + PendingRetransmissionState state = GetPendingState(subscriptionId); + state.NextSequenceNumber = nextSequenceNumber; + state.StateDirty = true; + + var liveSequences = new HashSet(); + foreach (NotificationMessage message in sentMessages) + { + liveSequences.Add(message.SequenceNumber); + if (state.KnownMessages.Add(message.SequenceNumber)) + { + state.PendingMessages[message.SequenceNumber] = message; + } + state.PendingDeletes.Remove(message.SequenceNumber); + } + + foreach (uint known in state.KnownMessages.ToArray()) + { + if (!liveSequences.Contains(known)) + { + state.KnownMessages.Remove(known); + state.PendingMessages.Remove(known); + state.PendingDeletes.Add(known); + } + } + } + + SignalDrain(); + } + + /// + public void StoreRetransmissionStateDelta( + uint subscriptionId, + uint nextSequenceNumber, + ArrayOf addedMessages, + ArrayOf removedSequenceNumbers) + { + lock (m_retransmissionLock) + { + PendingRetransmissionState state = GetPendingState(subscriptionId); + state.NextSequenceNumber = nextSequenceNumber; + state.StateDirty = true; + + foreach (uint sequenceNumber in removedSequenceNumbers) + { + state.KnownMessages.Remove(sequenceNumber); + state.PendingMessages.Remove(sequenceNumber); + state.PendingDeletes.Add(sequenceNumber); + } + + foreach (NotificationMessage message in addedMessages) + { + state.KnownMessages.Add(message.SequenceNumber); + state.PendingMessages[message.SequenceNumber] = message; + state.PendingDeletes.Remove(message.SequenceNumber); + } + } + + SignalDrain(); + } + + /// + /// Flushes queued retransmission mirror commands. + /// + /// The cancellation token. + internal async ValueTask FlushAsync(CancellationToken cancellationToken = default) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await m_channel.Writer + .WriteAsync(new MirrorCommand(completion), cancellationToken) + .ConfigureAwait(false); + await completion.Task.ConfigureAwait(false); + } + + /// + public async ValueTask DisposeAsync() + { + m_channel.Writer.TryComplete(); + await m_drainTask.ConfigureAwait(false); + m_drainCts.Dispose(); + } + + private PendingRetransmissionState GetPendingState(uint subscriptionId) + { + if (!m_pendingRetransmission.TryGetValue(subscriptionId, out PendingRetransmissionState? state)) + { + state = new PendingRetransmissionState(); + m_pendingRetransmission[subscriptionId] = state; + } + return state; + } + + /// + public void AcknowledgeNotification(uint subscriptionId, uint sequenceNumber) + { + lock (m_retransmissionLock) + { + PendingRetransmissionState state = GetPendingState(subscriptionId); + state.KnownMessages.Remove(sequenceNumber); + state.PendingMessages.Remove(sequenceNumber); + state.PendingDeletes.Add(sequenceNumber); + } + + SignalDrain(); + } + + /// + public void StoreContinuationPoint(ContinuationPointEnvelope envelope) + { + if (envelope == null) + { + throw new ArgumentNullException(nameof(envelope)); + } + + lock (m_continuationPointLock) + { + string key = ContinuationPointKeyFor(envelope.OwnerSessionId, envelope.Kind, envelope.Id); + m_pendingContinuationPointStores[key] = envelope; + m_pendingContinuationPointDeletes.Remove( + key); + } + + SignalDrain(); + } + + /// + public void RemoveContinuationPoint(NodeId ownerSessionId, ContinuationPointKind kind, Guid id) + { + if (ownerSessionId.IsNull) + { + return; + } + + string key = ContinuationPointKeyFor(ownerSessionId, kind, id); + lock (m_continuationPointLock) + { + m_pendingContinuationPointStores.Remove(key); + m_pendingContinuationPointDeletes.Add(key); + } + + SignalDrain(); + } + + /// + public async ValueTask> LoadContinuationPointsAsync( + NodeId ownerSessionId, + CancellationToken cancellationToken = default) + { + if (ownerSessionId.IsNull) + { + return []; + } + + var envelopes = new List(); + await foreach (KeyValuePair pair in m_store + .ScanAsync(ContinuationPointPrefixFor(ownerSessionId), cancellationToken) + .ConfigureAwait(false)) + { + if (m_protector.TryUnprotect(pair.Value, out ByteString payload)) + { + ContinuationPointEnvelope? envelope = DecodeContinuationPointEnvelope(payload); + if (envelope != null) + { + envelopes.Add(envelope); + } + } + } + return [.. envelopes]; + } + + private void SignalDrain() + { + if (m_channel.Writer.TryWrite(MirrorCommand.Signal)) + { + return; + } + + if (Interlocked.Exchange(ref m_overflowWarningWritten, 1) == 0) + { + m_logger?.LogWarning( + "The shared-state mirror channel is full; updates are coalesced until the drain catches up."); + } + } + + private void DeleteRetransmissionState(uint subscriptionId) + { + lock (m_retransmissionLock) + { + PendingRetransmissionState state = GetPendingState(subscriptionId); + state.ClearRequested = true; + state.StateDirty = false; + state.KnownMessages.Clear(); + state.PendingMessages.Clear(); + state.PendingDeletes.Clear(); + } + + SignalDrain(); + } + + private async Task DrainAsync() + { + await foreach (MirrorCommand command in m_channel.Reader + .ReadAllAsync(m_drainCts.Token) + .ConfigureAwait(false)) + { + try + { + await DrainPendingAsync(m_drainCts.Token).ConfigureAwait(false); + command.Completion?.SetResult(true); + } + catch (Exception ex) + { + m_logger?.LogWarning(ex, "Failed to mirror subscription retransmission state."); + command.Completion?.SetException(ex); + } + } + } + + private async ValueTask DrainPendingAsync(CancellationToken cancellationToken) + { + List batches = TakePendingBatches(); + foreach (RetransmissionBatch batch in batches) + { + try + { + if (batch.ClearRequested) + { + await m_store.DeleteAsync( + RetransmissionStateKeyFor(batch.SubscriptionId), + cancellationToken) + .ConfigureAwait(false); + await foreach (KeyValuePair pair in m_store + .ScanAsync(RetransmissionMessagePrefixFor(batch.SubscriptionId), cancellationToken) + .ConfigureAwait(false)) + { + await m_store.DeleteAsync(pair.Key, cancellationToken).ConfigureAwait(false); + } + } + + if (batch.StateDirty) + { + await m_store.SetAsync( + RetransmissionStateKeyFor(batch.SubscriptionId), + m_protector.Protect(EncodeRetransmissionState(batch.NextSequenceNumber)), + cancellationToken) + .ConfigureAwait(false); + } + + var operations = new List(batch.Messages.Length + batch.Deletes.Length); + foreach (NotificationMessage message in batch.Messages) + { + operations.Add(m_store.SetAsync( + RetransmissionMessageKeyFor(batch.SubscriptionId, message.SequenceNumber), + m_protector.Protect(EncodeNotificationMessage(message)), + cancellationToken) + .AsTask()); + } + + foreach (uint sequenceNumber in batch.Deletes) + { + operations.Add(m_store.DeleteAsync( + RetransmissionMessageKeyFor(batch.SubscriptionId, sequenceNumber), + cancellationToken) + .AsTask()); + } + await RunBatchOperationsAsync(operations).ConfigureAwait(false); + } + catch + { + Requeue(batch); + throw; + } + } + + ContinuationPointBatch continuationPointBatch = TakePendingContinuationPointBatch(); + try + { + foreach (ContinuationPointEnvelope envelope in continuationPointBatch.Stores) + { + await m_store.SetAsync( + ContinuationPointKeyFor(envelope.OwnerSessionId, envelope.Kind, envelope.Id), + m_protector.Protect(EncodeContinuationPointEnvelope(envelope)), + cancellationToken) + .ConfigureAwait(false); + } + + foreach (string key in continuationPointBatch.Deletes) + { + await m_store.DeleteAsync(key, cancellationToken).ConfigureAwait(false); + } + } + catch + { + Requeue(continuationPointBatch); + throw; + } + } + + private List TakePendingBatches() + { + lock (m_retransmissionLock) + { + var batches = new List(m_pendingRetransmission.Count); + var completedClears = new List(); + foreach (KeyValuePair entry in m_pendingRetransmission) + { + uint subscriptionId = entry.Key; + PendingRetransmissionState state = entry.Value; + if (!state.ClearRequested && + !state.StateDirty && + state.PendingMessages.Count == 0 && + state.PendingDeletes.Count == 0) + { + continue; + } + + batches.Add(new RetransmissionBatch( + subscriptionId, + state.NextSequenceNumber, + state.StateDirty, + state.PendingMessages.Values.ToArray(), + state.PendingDeletes.ToArray(), + state.ClearRequested)); + if (state.ClearRequested && + !state.StateDirty && + state.PendingMessages.Count == 0) + { + completedClears.Add(subscriptionId); + } + state.ClearRequested = false; + state.StateDirty = false; + state.PendingMessages.Clear(); + state.PendingDeletes.Clear(); + } + + foreach (uint subscriptionId in completedClears) + { + m_pendingRetransmission.Remove(subscriptionId); + } + + return batches; + } + } + + private void Requeue(RetransmissionBatch batch) + { + lock (m_retransmissionLock) + { + PendingRetransmissionState state = GetPendingState(batch.SubscriptionId); + state.ClearRequested |= batch.ClearRequested; + state.NextSequenceNumber = batch.NextSequenceNumber; + state.StateDirty |= batch.StateDirty; + foreach (NotificationMessage message in batch.Messages) + { + state.PendingMessages[message.SequenceNumber] = message; + } + foreach (uint sequenceNumber in batch.Deletes) + { + state.PendingDeletes.Add(sequenceNumber); + } + } + + SignalDrain(); + } + + private ContinuationPointBatch TakePendingContinuationPointBatch() + { + lock (m_continuationPointLock) + { + var batch = new ContinuationPointBatch( + m_pendingContinuationPointStores.Values.ToArray(), + m_pendingContinuationPointDeletes.ToArray()); + m_pendingContinuationPointStores.Clear(); + m_pendingContinuationPointDeletes.Clear(); + return batch; + } + } + + private void Requeue(ContinuationPointBatch batch) + { + lock (m_continuationPointLock) + { + foreach (ContinuationPointEnvelope envelope in batch.Stores) + { + m_pendingContinuationPointStores[ + ContinuationPointKeyFor(envelope.OwnerSessionId, envelope.Kind, envelope.Id)] = envelope; + } + foreach (string key in batch.Deletes) + { + m_pendingContinuationPointDeletes.Add(key); + } + } + + SignalDrain(); + } + + /// + /// Computes the shared-store key for a subscription definition. + /// + /// The subscription id. + /// The store key. + internal static string KeyFor(uint subscriptionId) + { + return Prefix + subscriptionId.ToString("D", System.Globalization.CultureInfo.InvariantCulture); + } + + internal static string RetransmissionStateKeyFor(uint subscriptionId) + { + return RetransmissionPrefix + + subscriptionId.ToString("D", System.Globalization.CultureInfo.InvariantCulture) + + "/state"; + } + + internal static string RetransmissionMessageKeyFor(uint subscriptionId, uint sequenceNumber) + { + return RetransmissionMessagePrefixFor(subscriptionId) + + sequenceNumber.ToString("D10", System.Globalization.CultureInfo.InvariantCulture); + } + + internal static string ContinuationPointKeyFor( + NodeId ownerSessionId, + ContinuationPointKind kind, + Guid id) + { + return ContinuationPointPrefixFor(ownerSessionId) + + ((int)kind).ToString(System.Globalization.CultureInfo.InvariantCulture) + + "/" + + id.ToString("N", System.Globalization.CultureInfo.InvariantCulture); + } + + private static StoredSubscription CloneSubscription(IStoredSubscription subscription) + { + return new StoredSubscription + { + Id = subscription.Id, + IsDurable = subscription.IsDurable, + LifetimeCounter = subscription.LifetimeCounter, + MaxLifetimeCount = subscription.MaxLifetimeCount, + MaxKeepaliveCount = subscription.MaxKeepaliveCount, + MaxMessageCount = subscription.MaxMessageCount, + MaxNotificationsPerPublish = subscription.MaxNotificationsPerPublish, + PublishingInterval = subscription.PublishingInterval, + Priority = subscription.Priority, + LastSentMessage = subscription.LastSentMessage, + SequenceNumber = subscription.SequenceNumber, + UserIdentityToken = subscription.UserIdentityToken, + SentMessages = subscription.SentMessages ?? [], + MonitoredItems = subscription.MonitoredItems.Select(CloneMonitoredItem).ToList() + }; + } + + private static StoredMonitoredItem CloneMonitoredItem(IStoredMonitoredItem item) + { + return new StoredMonitoredItem + { + IsRestored = item.IsRestored, + AlwaysReportUpdates = item.AlwaysReportUpdates, + AttributeId = item.AttributeId, + ClientHandle = item.ClientHandle, + DiagnosticsMasks = item.DiagnosticsMasks, + DiscardOldest = item.DiscardOldest, + Encoding = item.Encoding, + Id = item.Id, + IndexRange = item.IndexRange, + ParsedIndexRange = item.ParsedIndexRange, + IsDurable = item.IsDurable, + LastError = item.LastError, + LastValue = item.LastValue, + MonitoringMode = item.MonitoringMode, + NodeId = item.NodeId, + FilterToUse = item.FilterToUse, + OriginalFilter = item.OriginalFilter, + QueueSize = item.QueueSize, + Range = item.Range, + SamplingInterval = item.SamplingInterval, + SourceSamplingInterval = item.SourceSamplingInterval, + SubscriptionId = item.SubscriptionId, + TimestampsToReturn = item.TimestampsToReturn, + TypeMask = item.TypeMask + }; + } + + private static async ValueTask RunBatchOperationsAsync(List operations) + { + if (operations.Count == 0) + { + return; + } + + await Task.WhenAll(operations).ConfigureAwait(false); + } + + private ByteString Encode(StoredSubscription subscription) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteInt32(null, DefinitionFormatVersion); + encoder.WriteStringArray(null, m_context.NamespaceUris.ToArrayOf()); + encoder.WriteStringArray(null, m_context.ServerUris.ToArrayOf()); + EncodeSubscription(encoder, subscription); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : ByteString.From(buffer); + } + + private ByteString EncodeRetransmissionState(uint nextSequenceNumber) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteInt32(null, RetransmissionStateFormatVersion); + encoder.WriteUInt32(null, nextSequenceNumber); + encoder.WriteStringArray(null, m_context.NamespaceUris.ToArrayOf()); + encoder.WriteStringArray(null, m_context.ServerUris.ToArrayOf()); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : ByteString.From(buffer); + } + + private ByteString EncodeNotificationMessage(NotificationMessage message) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteInt32(null, NotificationMessageFormatVersion); + encoder.WriteEncodeable(null, message); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : ByteString.From(buffer); + } + + private ByteString EncodeContinuationPointEnvelope(ContinuationPointEnvelope envelope) + { + using var encoder = new BinaryEncoder(m_context); + encoder.WriteInt32(null, ContinuationPointFormatVersion); + encoder.WriteByteString(null, ByteString.From(envelope.Id.ToByteArray())); + encoder.WriteNodeId(null, envelope.OwnerSessionId); + encoder.WriteEnumerated(null, envelope.Kind); + encoder.WriteNodeId(null, envelope.BrowseNodeId); + encoder.WriteBoolean(null, envelope.View != null); + if (envelope.View != null) + { + encoder.WriteEncodeable(null, envelope.View); + } + encoder.WriteUInt32(null, envelope.MaxResultsToReturn); + encoder.WriteEnumerated(null, envelope.BrowseDirection); + encoder.WriteNodeId(null, envelope.ReferenceTypeId); + encoder.WriteBoolean(null, envelope.IncludeSubtypes); + encoder.WriteUInt32(null, envelope.NodeClassMask); + encoder.WriteEnumerated(null, envelope.ResultMask); + encoder.WriteInt32(null, envelope.Index); + byte[]? buffer = encoder.CloseAndReturnBuffer(); + return buffer is null ? ByteString.Empty : ByteString.From(buffer); + } + + private NotificationMessage DecodeNotificationMessage( + ByteString payload, + NamespaceTable? namespaceUris, + StringTable? serverUris) + { + try + { + using var decoder = new BinaryDecoder(payload.ToArray(), m_context); + int version = decoder.ReadInt32(null); + if (version != NotificationMessageFormatVersion || + namespaceUris == null || + serverUris == null) + { + throw new ServiceResultException(StatusCodes.BadDecodingError); + } + decoder.SetMappingTables(namespaceUris, serverUris); + return decoder.ReadEncodeable(null); + } + catch (Exception ex) when (ex is ServiceResultException or ArgumentException or InvalidOperationException) + { + using var decoder = new BinaryDecoder(payload.ToArray(), m_context); + ArrayOf legacyNamespaceUris = decoder.ReadStringArray(null); + ArrayOf legacyServerUris = decoder.ReadStringArray(null); + decoder.SetMappingTables( + CreateNamespaceTable(legacyNamespaceUris), + CreateStringTable(legacyServerUris)); + return decoder.ReadEncodeable(null); + } + } + + private ContinuationPointEnvelope? DecodeContinuationPointEnvelope(ByteString payload) + { + using var decoder = new BinaryDecoder(payload.ToArray(), m_context); + int version = decoder.ReadInt32(null); + if (version != ContinuationPointFormatVersion) + { + return null; + } + + ByteString idBytes = decoder.ReadByteString(null); + if (idBytes.Length != 16) + { + return null; + } + + return new ContinuationPointEnvelope + { + Id = new Guid(idBytes.ToArray()), + OwnerSessionId = decoder.ReadNodeId(null), + Kind = decoder.ReadEnumerated(null), + BrowseNodeId = decoder.ReadNodeId(null), + View = decoder.ReadBoolean(null) ? decoder.ReadEncodeable(null) : null, + MaxResultsToReturn = decoder.ReadUInt32(null), + BrowseDirection = decoder.ReadEnumerated(null), + ReferenceTypeId = decoder.ReadNodeId(null), + IncludeSubtypes = decoder.ReadBoolean(null), + NodeClassMask = decoder.ReadUInt32(null), + ResultMask = decoder.ReadEnumerated(null), + Index = decoder.ReadInt32(null) + }; + } + + private StoredSubscription Decode(ByteString payload) + { + using var decoder = new BinaryDecoder(payload.ToArray(), m_context); + int version = decoder.ReadInt32(null); + if (version != DefinitionFormatVersion) + { + throw new ServiceResultException(StatusCodes.BadDecodingError, "Unsupported subscription record version."); + } + + var namespaceUris = decoder.ReadStringArray(null); + var serverUris = decoder.ReadStringArray(null); + decoder.SetMappingTables(CreateNamespaceTable(namespaceUris), CreateStringTable(serverUris)); + return DecodeSubscription(decoder); + } + + private static void EncodeSubscription(BinaryEncoder encoder, StoredSubscription subscription) + { + encoder.WriteUInt32(null, subscription.Id); + encoder.WriteBoolean(null, subscription.IsDurable); + encoder.WriteUInt32(null, subscription.LifetimeCounter); + encoder.WriteUInt32(null, subscription.MaxLifetimeCount); + encoder.WriteUInt32(null, subscription.MaxKeepaliveCount); + encoder.WriteUInt32(null, subscription.MaxMessageCount); + encoder.WriteUInt32(null, subscription.MaxNotificationsPerPublish); + encoder.WriteDouble(null, subscription.PublishingInterval); + encoder.WriteByte(null, subscription.Priority); + encoder.WriteInt32(null, subscription.LastSentMessage); + encoder.WriteUInt32(null, subscription.SequenceNumber); + encoder.WriteExtensionObject( + null, + subscription.UserIdentityToken != null + ? new ExtensionObject(subscription.UserIdentityToken) + : ExtensionObject.Null); + + List items = subscription.MonitoredItems + .Select(CloneMonitoredItem) + .ToList(); + encoder.WriteInt32(null, items.Count); + foreach (StoredMonitoredItem item in items) + { + EncodeMonitoredItem(encoder, item); + } + } + + private static StoredSubscription DecodeSubscription(BinaryDecoder decoder) + { + var subscription = new StoredSubscription + { + Id = decoder.ReadUInt32(null), + IsDurable = decoder.ReadBoolean(null), + LifetimeCounter = decoder.ReadUInt32(null), + MaxLifetimeCount = decoder.ReadUInt32(null), + MaxKeepaliveCount = decoder.ReadUInt32(null), + MaxMessageCount = decoder.ReadUInt32(null), + MaxNotificationsPerPublish = decoder.ReadUInt32(null), + PublishingInterval = decoder.ReadDouble(null), + Priority = decoder.ReadByte(null), + LastSentMessage = decoder.ReadInt32(null), + SequenceNumber = decoder.ReadUInt32(null), + SentMessages = [] + }; + + ExtensionObject token = decoder.ReadExtensionObject(null); + if (!token.IsNull && + token.TryGetValue(out IEncodeable? tokenBody) && + tokenBody is UserIdentityToken userIdentityToken) + { + subscription.UserIdentityToken = userIdentityToken; + } + + int itemCount = decoder.ReadInt32(null); + var items = new List(itemCount); + for (int ii = 0; ii < itemCount; ii++) + { + items.Add(DecodeMonitoredItem(decoder)); + } + subscription.MonitoredItems = items; + return subscription; + } + + private static void EncodeMonitoredItem(BinaryEncoder encoder, StoredMonitoredItem item) + { + encoder.WriteBoolean(null, item.IsRestored); + encoder.WriteBoolean(null, item.AlwaysReportUpdates); + encoder.WriteUInt32(null, item.AttributeId); + encoder.WriteUInt32(null, item.ClientHandle); + encoder.WriteEnumerated(null, item.DiagnosticsMasks); + encoder.WriteBoolean(null, item.DiscardOldest); + encoder.WriteQualifiedName(null, item.Encoding); + encoder.WriteUInt32(null, item.Id); + encoder.WriteString(null, item.IndexRange); + encoder.WriteString(null, item.ParsedIndexRange.ToString()); + encoder.WriteBoolean(null, item.IsDurable); + encoder.WriteStatusCode(null, item.LastError?.StatusCode ?? StatusCodes.Good); + encoder.WriteDataValue(null, item.LastValue); + encoder.WriteEnumerated(null, item.MonitoringMode); + encoder.WriteNodeId(null, item.NodeId); + EncodeFilter(encoder, item.FilterToUse); + EncodeFilter(encoder, item.OriginalFilter); + encoder.WriteUInt32(null, item.QueueSize); + encoder.WriteDouble(null, item.Range); + encoder.WriteDouble(null, item.SamplingInterval); + encoder.WriteInt32(null, item.SourceSamplingInterval); + encoder.WriteUInt32(null, item.SubscriptionId); + encoder.WriteEnumerated(null, item.TimestampsToReturn); + encoder.WriteInt32(null, item.TypeMask); + } + + private static StoredMonitoredItem DecodeMonitoredItem(BinaryDecoder decoder) + { + var item = new StoredMonitoredItem + { + IsRestored = decoder.ReadBoolean(null), + AlwaysReportUpdates = decoder.ReadBoolean(null), + AttributeId = decoder.ReadUInt32(null), + ClientHandle = decoder.ReadUInt32(null), + DiagnosticsMasks = decoder.ReadEnumerated(null), + DiscardOldest = decoder.ReadBoolean(null), + Encoding = decoder.ReadQualifiedName(null), + Id = decoder.ReadUInt32(null), + IndexRange = decoder.ReadString(null) ?? string.Empty + }; + + string? rangeText = decoder.ReadString(null); + item.ParsedIndexRange = string.IsNullOrEmpty(rangeText) ? NumericRange.Null : NumericRange.Parse(rangeText); + item.IsDurable = decoder.ReadBoolean(null); + StatusCode lastError = decoder.ReadStatusCode(null); + item.LastError = lastError == StatusCodes.Good ? null! : new ServiceResult(lastError); + item.LastValue = decoder.ReadDataValue(null); + item.MonitoringMode = decoder.ReadEnumerated(null); + item.NodeId = decoder.ReadNodeId(null); + + item.FilterToUse = DecodeFilter(decoder); + item.OriginalFilter = DecodeFilter(decoder); + + item.QueueSize = decoder.ReadUInt32(null); + item.Range = decoder.ReadDouble(null); + item.SamplingInterval = decoder.ReadDouble(null); + item.SourceSamplingInterval = decoder.ReadInt32(null); + item.SubscriptionId = decoder.ReadUInt32(null); + item.TimestampsToReturn = decoder.ReadEnumerated(null); + item.TypeMask = decoder.ReadInt32(null); + return item; + } + + private static void EncodeFilter(BinaryEncoder encoder, MonitoringFilter? filter) + { + switch (filter) + { + case null: + encoder.WriteInt32(null, FilterKindNone); + break; + case DataChangeFilter dataChangeFilter: + encoder.WriteInt32(null, FilterKindDataChange); + encoder.WriteEncodeable(null, dataChangeFilter); + break; + case EventFilter eventFilter: + encoder.WriteInt32(null, FilterKindEvent); + encoder.WriteEncodeable(null, eventFilter); + break; + case AggregateFilter aggregateFilter: + encoder.WriteInt32(null, FilterKindAggregate); + encoder.WriteEncodeable(null, aggregateFilter); + break; + default: + encoder.WriteInt32(null, FilterKindExtensionObject); + encoder.WriteExtensionObject(null, new ExtensionObject(filter)); + break; + } + } + + private static MonitoringFilter DecodeFilter(BinaryDecoder decoder) + { + int kind = decoder.ReadInt32(null); + return kind switch + { + FilterKindNone => null!, + FilterKindDataChange => decoder.ReadEncodeable(null), + FilterKindEvent => decoder.ReadEncodeable(null), + FilterKindAggregate => decoder.ReadEncodeable(null), + FilterKindExtensionObject => DecodeExtensionObjectFilter(decoder), + _ => throw new ServiceResultException(StatusCodes.BadDecodingError, "Unsupported monitoring filter kind.") + }; + } + + private static MonitoringFilter DecodeExtensionObjectFilter(BinaryDecoder decoder) + { + ExtensionObject filter = decoder.ReadExtensionObject(null); + if (!filter.IsNull && + filter.TryGetValue(out IEncodeable? filterBody) && + filterBody is MonitoringFilter monitoringFilter) + { + return monitoringFilter; + } + + return null!; + } + + private static NamespaceTable CreateNamespaceTable(ArrayOf namespaceUris) + { + return new NamespaceTable(namespaceUris.Memory.ToArray().Where(s => s != null).Select(s => s!).ToArray()); + } + + private static StringTable CreateStringTable(ArrayOf serverUris) + { + return new StringTable(serverUris.Memory.ToArray().Where(s => s != null).Select(s => s!).ToArray()); + } + + private static string RetransmissionMessagePrefixFor(uint subscriptionId) + { + return RetransmissionPrefix + + subscriptionId.ToString("D", System.Globalization.CultureInfo.InvariantCulture) + + "/message/"; + } + + private static string ContinuationPointPrefixFor(NodeId ownerSessionId) + { + return ContinuationPointPrefix + + Uri.EscapeDataString(ownerSessionId.ToString()) + + "/"; + } + + private const int DefinitionFormatVersion = 1; + private const int ContinuationPointFormatVersion = 1; + private const int LegacyRetransmissionStateFormatVersion = 1; + private const int RetransmissionStateFormatVersion = 2; + private const int NotificationMessageFormatVersion = 2; + private const int FilterKindNone = 0; + private const int FilterKindDataChange = 1; + private const int FilterKindEvent = 2; + private const int FilterKindAggregate = 3; + private const int FilterKindExtensionObject = 4; + private const int ChannelCapacity = 1024; + private const string Prefix = "subscription/"; + private const string RetransmissionPrefix = "subscription-retransmission/"; + private const string ContinuationPointPrefix = "continuation-point/"; + private readonly ISharedKeyValueStore m_store; + private readonly IServiceMessageContext m_context; + private readonly IRecordProtector m_protector; + private readonly ILogger? m_logger; + private readonly SharedDefinitionCache m_definitionCache; + private readonly Channel m_channel; + private readonly CancellationTokenSource m_drainCts = new(); + private readonly Task m_drainTask; + private readonly object m_retransmissionLock = new(); + private readonly object m_continuationPointLock = new(); + private readonly Dictionary m_pendingRetransmission = []; + private readonly Dictionary m_pendingContinuationPointStores = []; + private readonly HashSet m_pendingContinuationPointDeletes = []; + private static readonly ConditionalWeakTable s_definitionCaches = + new(); + private int m_overflowWarningWritten; + + private sealed class SharedDefinitionCache + { + public object Lock { get; } = new(); + + public Dictionary Subscriptions { get; } = []; + } + + private sealed class PendingRetransmissionState + { + public uint NextSequenceNumber { get; set; } + + public bool ClearRequested { get; set; } + + public bool StateDirty { get; set; } + + public HashSet KnownMessages { get; } = []; + + public Dictionary PendingMessages { get; } = []; + + public HashSet PendingDeletes { get; } = []; + } + + private readonly record struct RetransmissionBatch( + uint SubscriptionId, + uint NextSequenceNumber, + bool StateDirty, + NotificationMessage[] Messages, + uint[] Deletes, + bool ClearRequested); + + private readonly record struct ContinuationPointBatch( + ContinuationPointEnvelope[] Stores, + string[] Deletes); + + private sealed class MirrorCommand + { + public static MirrorCommand Signal { get; } = new(null); + + public MirrorCommand(TaskCompletionSource? completion) + { + Completion = completion; + } + + public TaskCompletionSource? Completion { get; } + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Values/DistributedValueCache.cs b/Libraries/Opc.Ua.Redundancy.Server/Values/DistributedValueCache.cs new file mode 100644 index 0000000000..0701ed9116 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Values/DistributedValueCache.cs @@ -0,0 +1,87 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: default over an + /// . Freshness is evaluated against the + /// value's using an injectable + /// . + /// + public sealed class DistributedValueCache : IDistributedValueCache + { + /// + /// Creates a value cache over a node state store. + /// + /// The backing node state store. + /// Time source (defaults to system). + public DistributedValueCache(INodeStateStore store, TimeProvider? timeProvider = null) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + m_time = timeProvider ?? TimeProvider.System; + } + + /// + public ValueTask CacheAsync(NodeId nodeId, in DataValue value, CancellationToken ct = default) + { + return m_store.WriteValueAsync(nodeId, value, ct); + } + + /// + public async ValueTask<(bool Fresh, DataValue Value)> TryGetAsync( + NodeId nodeId, + TimeSpan maxAge, + CancellationToken ct = default) + { + (bool found, DataValue value) = await m_store + .TryReadValueAsync(nodeId, ct) + .ConfigureAwait(false); + if (!found) + { + return (false, DataValue.Null); + } + + DateTimeUtc now = m_time.GetUtcNow(); + // A missing timestamp (MinValue) yields a huge age and is treated + // as stale; clock skew into the future yields a negative age and + // is treated as fresh. + bool fresh = (now - value.SourceTimestamp) <= maxAge; + return (fresh, value); + } + + private readonly INodeStateStore m_store; + private readonly TimeProvider m_time; + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Values/DistributedValueParticipation.cs b/Libraries/Opc.Ua.Redundancy.Server/Values/DistributedValueParticipation.cs new file mode 100644 index 0000000000..4d8f3c3576 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Values/DistributedValueParticipation.cs @@ -0,0 +1,132 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: helpers that let a variable's read/write callbacks participate in the + /// distributed value cache: serve the last value with a freshness bound + /// and cache values on read/write. Monitored items are unaffected — they + /// continue to read through the normal pipeline and therefore observe the + /// cached value only when the read path participates. + /// + public static class DistributedValueParticipation + { + /// + /// Returns the cached value when it is fresh; otherwise invokes + /// , caches the result, and returns it. + /// Use from a custom read callback. + /// + /// The distributed value cache. + /// The variable node identifier. + /// The freshness bound for the cached value. + /// Reads the live value from the source. + /// Cancellation token. + public static async ValueTask ReadThroughAsync( + IDistributedValueCache cache, + NodeId nodeId, + TimeSpan maxAge, + Func> liveRead, + CancellationToken ct = default) + { + if (cache == null) + { + throw new ArgumentNullException(nameof(cache)); + } + if (liveRead == null) + { + throw new ArgumentNullException(nameof(liveRead)); + } + + (bool fresh, DataValue cached) = await cache.TryGetAsync(nodeId, maxAge, ct).ConfigureAwait(false); + if (fresh) + { + return cached; + } + + DataValue live = await liveRead(ct).ConfigureAwait(false); + await cache.CacheAsync(nodeId, live, ct).ConfigureAwait(false); + return live; + } + + /// + /// Wires a variable's asynchronous read/write callbacks to the + /// distributed value cache: reads serve the last value while fresh + /// (falling back to and caching), and + /// writes are cached (write-through). + /// + /// The variable to wire. + /// The distributed value cache. + /// The freshness bound for cached reads. + /// Reads the live value from the source. + public static void EnableDistributedValueParticipation( + this BaseVariableState variable, + IDistributedValueCache cache, + TimeSpan maxAge, + Func> liveRead) + { + if (variable == null) + { + throw new ArgumentNullException(nameof(variable)); + } + if (cache == null) + { + throw new ArgumentNullException(nameof(cache)); + } + if (liveRead == null) + { + throw new ArgumentNullException(nameof(liveRead)); + } + + NodeId nodeId = variable.NodeId; + + variable.OnReadValueAsync = async (context, node, indexRange, dataEncoding, ct) => + { + DataValue value = await ReadThroughAsync(cache, nodeId, maxAge, liveRead, ct).ConfigureAwait(false); + return new AttributeReadResult( + ServiceResult.Good, + value.WrappedValue, + value.StatusCode, + value.SourceTimestamp); + }; + + variable.OnWriteValueAsync = async (context, node, indexRange, value, ct) => + { + var dataValue = new DataValue(value, StatusCodes.Good, DateTimeUtc.Now); + await cache.CacheAsync(nodeId, dataValue, ct).ConfigureAwait(false); + return new AttributeWriteResult(ServiceResult.Good); + }; + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy.Server/Values/IDistributedValueCache.cs b/Libraries/Opc.Ua.Redundancy.Server/Values/IDistributedValueCache.cs new file mode 100644 index 0000000000..18c53fb79a --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy.Server/Values/IDistributedValueCache.cs @@ -0,0 +1,73 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server +{ + /// + /// Extension beyond OPC 10000-4 §6.6: a read/write value cache backed by a distributed + /// . + /// + /// + /// OPC 10000-4 §6.6.2.2 requires identical NodeIds and AddressSpaces in a RedundantServerSet, but it does + /// not standardize how process values are cached between replicas. This provider lets variable callbacks cache the + /// last value they observed and serve it with a freshness bound from the shared store. + /// + public interface IDistributedValueCache + { + /// + /// Writes the latest value for a node into the cache. + /// + /// The variable node identifier. + /// The value to cache. + /// Cancellation token. + ValueTask CacheAsync(NodeId nodeId, in DataValue value, CancellationToken ct = default); + + /// + /// Reads the last cached value and reports whether it is still fresh + /// (its source timestamp is within ). + /// + /// The variable node identifier. + /// The freshness bound. + /// Cancellation token. + /// + /// Fresh = true with the value when a value exists and is + /// within ; otherwise the value may still be + /// returned (when present) but Fresh = false. + /// + ValueTask<(bool Fresh, DataValue Value)> TryGetAsync( + NodeId nodeId, + TimeSpan maxAge, + CancellationToken ct = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy/ByteStringCrdtSerializer.cs b/Libraries/Opc.Ua.Redundancy/ByteStringCrdtSerializer.cs new file mode 100644 index 0000000000..d3ff04a8f1 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/ByteStringCrdtSerializer.cs @@ -0,0 +1,99 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Text.Json; +using Crdt; + +namespace Opc.Ua.Redundancy +{ + /// + /// CRDT value serializer for payloads (serialized + /// node states and encoded values). A leading var-uint marker carries + /// 0 for a null and length + 1 + /// otherwise, so null and empty are distinguished. + /// + internal sealed class ByteStringCrdtSerializer : ICrdtValueSerializer + { + /// + /// The shared serializer instance. + /// + public static ByteStringCrdtSerializer Instance { get; } = new(); + + /// + public void Write(ref CrdtWriter writer, ByteString value) + { + if (value.IsNull) + { + writer.WriteVarUInt32(0); + return; + } + + byte[] bytes = value.ToArray(); + writer.WriteVarUInt32((uint)bytes.Length + 1); + writer.WriteRaw(bytes); + } + + /// + public ByteString Read(ref CrdtReader reader) + { + uint marker = reader.ReadVarUInt32(); + if (marker == 0) + { + return default; + } + + ReadOnlySpan raw = reader.ReadRaw((int)(marker - 1)); + return new ByteString(raw.ToArray()); + } + + /// + public void WriteJson(Utf8JsonWriter writer, ByteString value) + { + if (value.IsNull) + { + writer.WriteNullValue(); + return; + } + + writer.WriteBase64StringValue(value.ToArray()); + } + + /// + public ByteString ReadJson(ref Utf8JsonReader reader) + { + if (reader.TokenType == JsonTokenType.Null) + { + return default; + } + + return new ByteString(reader.GetBytesFromBase64()); + } + } +} diff --git a/Libraries/Opc.Ua.Redundancy/CrdtSharedKeyValueStore.cs b/Libraries/Opc.Ua.Redundancy/CrdtSharedKeyValueStore.cs new file mode 100644 index 0000000000..368673320c --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/CrdtSharedKeyValueStore.cs @@ -0,0 +1,225 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: a CRDT-backed : a last-writer-wins map + /// gossiped between replicas so every replica converges on the same + /// key/value state without a leader. Reads, writes, and deletes are local; + /// the merged state propagates by gossip. + /// + /// + /// CRDTs are eventually consistent (AP) and cannot provide a linearizable + /// compare-and-swap, so is not supported. + /// Use a strongly-consistent store for primitives that require exactly-once + /// semantics (for example the single-use session nonce registry). + /// + public sealed class CrdtSharedKeyValueStore : ISharedKeyValueStore, IAsyncDisposable + { + /// + /// Creates a CRDT key/value store. + /// + /// This replica's stable CRDT identity. + /// The gossip transport (owned by this store). + /// The time source for the logical clock. + /// Decoding limits for received state. + public CrdtSharedKeyValueStore( + ReplicaId replicaId, + ITransport transport, + TimeProvider timeProvider, + CrdtReaderOptions readerOptions) + { + m_transport = transport ?? throw new ArgumentNullException(nameof(transport)); + m_readerOptions = readerOptions ?? throw new ArgumentNullException(nameof(readerOptions)); + m_clock = new HybridLogicalClock( + replicaId, + timeProvider ?? throw new ArgumentNullException(nameof(timeProvider))); + m_transport.FrameReceived += OnFrameReceived; + } + + /// + public async ValueTask<(bool Found, ByteString Value)> TryGetAsync(string key, CancellationToken ct = default) + { + await EnsureStartedAsync(ct).ConfigureAwait(false); + lock (m_lock) + { + return m_map.TryGetValue(key, out ByteString value) ? (true, value) : (false, default); + } + } + + /// + public async ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) + { + await EnsureStartedAsync(ct).ConfigureAwait(false); + byte[] snapshot; + lock (m_lock) + { + m_map.Set(key, value, m_clock); + snapshot = SerializeLocked(); + } + await m_transport.SendAsync(snapshot, ct).ConfigureAwait(false); + } + + /// + public async ValueTask CompareAndSwapAsync( + string key, + ByteString expected, + ByteString value, + CancellationToken ct = default) + { + await EnsureStartedAsync(ct).ConfigureAwait(false); + throw new NotSupportedException( + "CrdtSharedKeyValueStore is eventually consistent and does not support compare-and-swap. " + + "Use a strongly-consistent store for compare-and-swap primitives (e.g. the single-use nonce registry)."); + } + + /// + public async ValueTask DeleteAsync(string key, CancellationToken ct = default) + { + await EnsureStartedAsync(ct).ConfigureAwait(false); + byte[] snapshot; + bool existed; + lock (m_lock) + { + existed = m_map.ContainsKey(key); + m_map.Remove(key, m_clock); + snapshot = SerializeLocked(); + } + await m_transport.SendAsync(snapshot, ct).ConfigureAwait(false); + return existed; + } + + /// + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) + { + await EnsureStartedAsync(ct).ConfigureAwait(false); + List> snapshot = []; + lock (m_lock) + { + foreach (string key in m_map.Keys) + { + if (key.StartsWith(keyPrefix, StringComparison.Ordinal) && + m_map.TryGetValue(key, out ByteString value)) + { + snapshot.Add(new KeyValuePair(key, value)); + } + } + } + + foreach (KeyValuePair pair in snapshot) + { + ct.ThrowIfCancellationRequested(); + yield return pair; + } + } + + /// + public IAsyncEnumerable WatchAsync(string keyPrefix, CancellationToken ct = default) + { + throw new NotSupportedException( + "CrdtSharedKeyValueStore does not expose a change feed; it is intended for entry replication " + + "(for example mirrored session entries) where consumers read on demand."); + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_lock) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + + m_transport.FrameReceived -= OnFrameReceived; + await m_transport.DisposeAsync().ConfigureAwait(false); + m_startGate.Dispose(); + } + + private async ValueTask EnsureStartedAsync(CancellationToken ct) + { + if (Volatile.Read(ref m_started)) + { + return; + } + + await m_startGate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (!m_started) + { + await m_transport.StartAsync(ct).ConfigureAwait(false); + Volatile.Write(ref m_started, true); + } + } + finally + { + m_startGate.Release(); + } + } + + private void OnFrameReceived(ReadOnlyMemory frame) + { + byte[] bytes = frame.ToArray(); + lock (m_lock) + { + LWWMap remote = LWWMap.ReadFrom( + bytes, CrdtValues.String, ByteStringCrdtSerializer.Instance, m_readerOptions); + m_map.Merge(remote); + } + } + + private byte[] SerializeLocked() + { + return m_map.ToByteArray(CrdtValues.String, ByteStringCrdtSerializer.Instance); + } + + private readonly ITransport m_transport; + private readonly CrdtReaderOptions m_readerOptions; + private readonly HybridLogicalClock m_clock; + private readonly Lock m_lock = new(); + private readonly LWWMap m_map = new(); + private readonly SemaphoreSlim m_startGate = new(1, 1); + private bool m_started; + private bool m_disposed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy/NugetREADME.md b/Libraries/Opc.Ua.Redundancy/NugetREADME.md new file mode 100644 index 0000000000..99429c02df --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/NugetREADME.md @@ -0,0 +1,10 @@ +# OPCFoundation.NetStandard.Opc.Ua.Redundancy + +Shared replication building blocks for OPC UA redundancy, used by both `Opc.Ua.Redundancy.Server` and `Opc.Ua.Redundancy.Client` and built on the `Opc.Ua.Redundancy` seams in `Opc.Ua.Core`. + +- **Eventually-consistent (AP):** `ByteStringCrdtSerializer` and `CrdtSharedKeyValueStore` — an `ISharedKeyValueStore` that gossips state between replicas without a leader. +- **Strongly-consistent (CP):** a Raft layer behind the `IRaftConsensus` seam — `RaftSharedKeyValueStore` (linearizable `CompareAndSwapAsync` + `WatchAsync`) and `RaftLeaderElection` (native single-leader election). The DI default is a single-node [`RaftCs`](https://github.com/marcschier/raft-cs) replica via `RaftCsConsensus`; `InProcessRaftConsensus` is a lighter deterministic in-process backend, and a multi-node `RaftNode` (NanoMsg transport + file WAL) plugs in for multi-pod clusters. +- **Hybrid:** `HybridSharedKeyValueStore` serves bulk keys from the CRDT store and the strong keyspaces (single-use nonces, lease, election) from Raft, selected with `RedundancyConsistencyMode`. + +See `Docs/HighAvailability.md` (*Consistency modes*) for guidance. + diff --git a/Libraries/Opc.Ua.Redundancy/Opc.Ua.Redundancy.csproj b/Libraries/Opc.Ua.Redundancy/Opc.Ua.Redundancy.csproj new file mode 100644 index 0000000000..ce51be138c --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Opc.Ua.Redundancy.csproj @@ -0,0 +1,34 @@ + + + $(LibTargetFrameworks) + $(PackagePrefix).Opc.Ua.Redundancy + $(AssemblyPrefix).Redundancy + Opc.Ua.Redundancy + OPC UA Redundancy shared CRDT building blocks (serializer + gossip key/value store) + PackageReference + true + NugetREADME.md + true + enable + true + $(DefineConstants);OPCUA_RAFTCS + + + + + + + $(PackageId).Debug + + + + + + + + + + + + + diff --git a/Libraries/Opc.Ua.Redundancy/Properties/AssemblyInfo.cs b/Libraries/Opc.Ua.Redundancy/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..6f476448c4 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Runtime.CompilerServices; + +[assembly: CLSCompliant(false)] +[assembly: InternalsVisibleTo("Opc.Ua.Aot.Tests")] +[assembly: InternalsVisibleTo("Opc.Ua.Redundancy.Server")] diff --git a/Libraries/Opc.Ua.Redundancy/Raft/HybridSharedKeyValueStore.cs b/Libraries/Opc.Ua.Redundancy/Raft/HybridSharedKeyValueStore.cs new file mode 100644 index 0000000000..d6f21505d1 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/HybridSharedKeyValueStore.cs @@ -0,0 +1,247 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: the store. It routes each + /// key to exactly one backend by key prefix: keys under a configured "strong" prefix (the single-use nonce, + /// lease, and leader-election keyspaces) live entirely in the linearizable ; + /// all other keys live in the eventually-consistent . + /// + /// + /// + /// The invariant is "a key lives in exactly one store", so a compare-and-swap or change-feed on a strong key gets + /// the linearizable Raft semantics, while bulk keys keep the leaderless CRDT semantics (and, as with the raw CRDT + /// store, do not support compare-and-swap / watch — by design those operations belong to the strong keyspace). + /// + /// + /// A scan whose prefix spans both keyspaces (for example the empty prefix used for full hydration) enumerates the + /// bulk store and then the strong store. + /// + /// + public sealed class HybridSharedKeyValueStore : ISharedKeyValueStore, IAsyncDisposable + { + /// + /// Creates a hybrid store. + /// + /// The eventually-consistent backend for bulk keys. + /// The linearizable backend for strong keys. + /// + /// The key prefixes routed to . When empty, the defaults + /// (nonce/, lease/, election/) are used. + /// + /// + /// When true, both backends are disposed together with this store. + /// + public HybridSharedKeyValueStore( + ISharedKeyValueStore bulkStore, + ISharedKeyValueStore strongStore, + ArrayOf strongKeyPrefixes = default, + bool ownsStores = false) + { + m_bulk = bulkStore ?? throw new ArgumentNullException(nameof(bulkStore)); + m_strong = strongStore ?? throw new ArgumentNullException(nameof(strongStore)); + m_ownsStores = ownsStores; + m_strongPrefixes = strongKeyPrefixes.IsEmpty ? s_defaultStrongPrefixes : ToArray(strongKeyPrefixes); + } + + /// + public ValueTask<(bool Found, ByteString Value)> TryGetAsync(string key, CancellationToken ct = default) + { + return Route(key).TryGetAsync(key, ct); + } + + /// + public ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) + { + return Route(key).SetAsync(key, value, ct); + } + + /// + public ValueTask CompareAndSwapAsync( + string key, + ByteString expected, + ByteString value, + CancellationToken ct = default) + { + return Route(key).CompareAndSwapAsync(key, expected, value, ct); + } + + /// + public ValueTask DeleteAsync(string key, CancellationToken ct = default) + { + return Route(key).DeleteAsync(key, ct); + } + + /// + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + keyPrefix ??= string.Empty; + + if (IsStrong(keyPrefix)) + { + await foreach (KeyValuePair entry in m_strong + .ScanAsync(keyPrefix, ct) + .ConfigureAwait(false)) + { + yield return entry; + } + yield break; + } + + await foreach (KeyValuePair entry in m_bulk + .ScanAsync(keyPrefix, ct) + .ConfigureAwait(false)) + { + yield return entry; + } + + if (SpansStrong(keyPrefix)) + { + await foreach (KeyValuePair entry in m_strong + .ScanAsync(keyPrefix, ct) + .ConfigureAwait(false)) + { + yield return entry; + } + } + } + + /// + public IAsyncEnumerable WatchAsync(string keyPrefix, CancellationToken ct = default) + { + keyPrefix ??= string.Empty; + return IsStrong(keyPrefix) + ? m_strong.WatchAsync(keyPrefix, ct) + : m_bulk.WatchAsync(keyPrefix, ct); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref m_disposed, 1) != 0) + { + return; + } + if (m_ownsStores) + { + await DisposeStoreAsync(m_strong).ConfigureAwait(false); + await DisposeStoreAsync(m_bulk).ConfigureAwait(false); + } + } + + private ISharedKeyValueStore Route(string key) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + return IsStrong(key) ? m_strong : m_bulk; + } + + /// + /// Returns whether is routed to the linearizable strong (Raft) store. + /// + /// The key (or key prefix) to test. + public bool IsStrongKey(string key) + { + return IsStrong(key ?? string.Empty); + } + + private bool IsStrong(string keyOrPrefix) + { + for (int ii = 0; ii < m_strongPrefixes.Length; ii++) + { + if (keyOrPrefix.StartsWith(m_strongPrefixes[ii], StringComparison.Ordinal)) + { + return true; + } + } + return false; + } + + private bool SpansStrong(string scanPrefix) + { + for (int ii = 0; ii < m_strongPrefixes.Length; ii++) + { + if (m_strongPrefixes[ii].StartsWith(scanPrefix, StringComparison.Ordinal)) + { + return true; + } + } + return false; + } + + private static async ValueTask DisposeStoreAsync(ISharedKeyValueStore store) + { + switch (store) + { + case IAsyncDisposable asyncDisposable: + await asyncDisposable.DisposeAsync().ConfigureAwait(false); + break; + case IDisposable disposable: + disposable.Dispose(); + break; + } + } + + private static string[] ToArray(ArrayOf prefixes) + { + var result = new string[prefixes.Count]; + for (int ii = 0; ii < prefixes.Count; ii++) + { + result[ii] = prefixes[ii]; + } + return result; + } + + private static readonly string[] s_defaultStrongPrefixes = ["nonce/", "lease/", "election/"]; + + /// + /// The default strong-keyspace prefixes (nonce/, lease/, election/) used when no explicit + /// set is configured. + /// + public static ArrayOf DefaultStrongKeyPrefixes { get; } = ["nonce/", "lease/", "election/"]; + + private readonly ISharedKeyValueStore m_bulk; + private readonly ISharedKeyValueStore m_strong; + private readonly string[] m_strongPrefixes; + private readonly bool m_ownsStores; + private int m_disposed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy/Raft/IRaftConsensus.cs b/Libraries/Opc.Ua.Redundancy/Raft/IRaftConsensus.cs new file mode 100644 index 0000000000..b81c6644af --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/IRaftConsensus.cs @@ -0,0 +1,95 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: the strongly-consistent (CP) consensus seam used to build a linearizable + /// and a native . + /// + /// + /// + /// The shape mirrors a replicated-state-machine consensus node: an opaque application command is + /// d, replicated, and — once committed — surfaced in log order on + /// so every replica applies the same sequence of commands. This is exactly the contract + /// of the RaftNode facade in the external RaftCs package (marcschier/raft-cs), so the + /// production adapter that wraps a real Raft replica is a thin shim over this interface. + /// + /// + /// provides a deterministic in-memory backend (a single shared committed log) + /// for single-process deployments, in-process replica sets, and tests; a multi-node Raft engine is a drop-in + /// replacement of the same contract. + /// + /// + public interface IRaftConsensus : IAsyncDisposable + { + /// + /// true when this replica currently believes itself to be the leader (the only replica that may + /// originate proposals in a real Raft cluster). + /// + bool IsLeader { get; } + + /// + /// Raised when leadership is gained (true) or lost (false). + /// + event Action? LeadershipChanged; + + /// + /// A reader over committed application command payloads, in log order. Every replica observes the identical + /// sequence, which is what makes a state machine built on top deterministic and linearizable. + /// + ChannelReader> Committed { get; } + + /// + /// Starts the consensus replica (joins the cluster / begins the driver loop). + /// + /// Cancellation token. + ValueTask StartAsync(CancellationToken ct = default); + + /// + /// Proposes an opaque application command to be replicated and committed. The command is surfaced on + /// once it commits. + /// + /// The opaque command payload. + /// Cancellation token. + ValueTask ProposeAsync(ReadOnlyMemory command, CancellationToken ct = default); + + /// + /// Requests that this replica (try to) become the leader. Best-effort: a backend with deterministic + /// leadership may treat this as a no-op and simply report the current state. + /// + /// Cancellation token. + ValueTask CampaignAsync(CancellationToken ct = default); + } +} diff --git a/Libraries/Opc.Ua.Redundancy/Raft/InProcessRaftCluster.cs b/Libraries/Opc.Ua.Redundancy/Raft/InProcessRaftCluster.cs new file mode 100644 index 0000000000..a72ca16728 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/InProcessRaftCluster.cs @@ -0,0 +1,137 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: an in-process coordinator that links one or more + /// replicas onto a single shared, totally-ordered committed log. Proposing on + /// any member broadcasts the command to every member in one global order, so a state machine built on top + /// (for example RaftSharedKeyValueStore) converges deterministically — the same guarantee a real + /// Raft cluster provides, without the network. + /// + /// + /// Leadership is deterministic: the live member with the lowest node id is the leader. Disposing the leader + /// re-elects the next-lowest member and raises , which makes + /// failover paths exercisable in a single process and in CI. + /// + public sealed class InProcessRaftCluster + { + /// + /// Creates a new consensus replica attached to this cluster. + /// + /// + /// This replica's unique, non-zero identity; the lowest live id is the leader. + /// + public InProcessRaftConsensus CreateNode(ulong nodeId) + { + return new InProcessRaftConsensus(this, nodeId); + } + + internal void Register(InProcessRaftConsensus node) + { + lock (m_lock) + { + if (!m_members.Contains(node)) + { + m_members.Add(node); + } + } + Reelect(); + } + + internal void Unregister(InProcessRaftConsensus node) + { + lock (m_lock) + { + m_members.Remove(node); + } + Reelect(); + } + + internal void Propose(ReadOnlyMemory command) + { + // Hold the lock across the whole broadcast so every member observes + // commands in one identical global order, even under concurrent + // proposers. Channel writes are non-blocking (unbounded). + lock (m_lock) + { + for (int ii = 0; ii < m_members.Count; ii++) + { + m_members[ii].Deliver(command); + } + } + } + + private void Reelect() + { + var transitions = new List<(InProcessRaftConsensus Node, bool IsLeader)>(); + lock (m_lock) + { + InProcessRaftConsensus? newLeader = null; + for (int ii = 0; ii < m_members.Count; ii++) + { + InProcessRaftConsensus candidate = m_members[ii]; + if (newLeader == null || candidate.NodeId < newLeader.NodeId) + { + newLeader = candidate; + } + } + + if (!ReferenceEquals(newLeader, m_leader)) + { + if (m_leader != null) + { + transitions.Add((m_leader, false)); + } + m_leader = newLeader; + if (newLeader != null) + { + transitions.Add((newLeader, true)); + } + } + } + + // Raise leadership callbacks outside the lock: never invoke consumer + // delegates while holding an internal lock. + foreach ((InProcessRaftConsensus node, bool isLeader) in transitions) + { + node.SetLeadership(isLeader); + } + } + + private readonly Lock m_lock = new(); + private readonly List m_members = []; + private InProcessRaftConsensus? m_leader; + } +} diff --git a/Libraries/Opc.Ua.Redundancy/Raft/InProcessRaftConsensus.cs b/Libraries/Opc.Ua.Redundancy/Raft/InProcessRaftConsensus.cs new file mode 100644 index 0000000000..eab5180319 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/InProcessRaftConsensus.cs @@ -0,0 +1,169 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: a deterministic, in-process replica. Used + /// standalone it forms a single-node "cluster" (always the leader, fully linearizable) for single-process + /// deployments and tests; attached to an it shares one totally-ordered + /// committed log with its peers so a multi-replica state machine converges without a network. + /// + /// + /// This is the offline / in-process backend behind RaftSharedKeyValueStore and + /// RaftLeaderElection. The external multi-node Raft engine (RaftCs) is a drop-in replacement + /// of the contract. + /// + public sealed class InProcessRaftConsensus : IRaftConsensus + { + /// + /// Creates a standalone single-node replica (its own private one-member cluster). The node is always the + /// leader once started. + /// + /// + /// This replica's unique, non-zero identity (defaults to 1). + /// + public InProcessRaftConsensus(ulong nodeId = 1) + : this(new InProcessRaftCluster(), nodeId) + { + } + + /// + /// Creates a replica attached to a shared . + /// + /// The shared in-process cluster. + /// + /// This replica's unique, non-zero identity; the lowest live id is the leader. + /// + public InProcessRaftConsensus(InProcessRaftCluster cluster, ulong nodeId) + { + m_cluster = cluster ?? throw new ArgumentNullException(nameof(cluster)); + NodeId = nodeId; + m_committed = Channel.CreateUnbounded>( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + } + + /// + /// This replica's unique identity within the cluster. + /// + public ulong NodeId { get; } + + /// + public bool IsLeader => Volatile.Read(ref m_isLeader); + + /// + public event Action? LeadershipChanged; + + /// + public ChannelReader> Committed => m_committed.Reader; + + /// + public ValueTask StartAsync(CancellationToken ct = default) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + if (Interlocked.Exchange(ref m_started, 1) == 0) + { + m_cluster.Register(this); + } + return default; + } + + /// + public ValueTask ProposeAsync(ReadOnlyMemory command, CancellationToken ct = default) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + + // Copy into a stable, immutable buffer so every replica shares an + // identical view independent of the caller's buffer lifetime. + byte[] copy = command.ToArray(); + m_cluster.Propose(copy); + return default; + } + + /// + public ValueTask CampaignAsync(CancellationToken ct = default) + { + ThrowIfDisposed(); + ct.ThrowIfCancellationRequested(); + + // Leadership is deterministic (lowest live id). Campaigning is a + // no-op here; the real Raft backend forces an election campaign. + return default; + } + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref m_disposed, 1) != 0) + { + return default; + } + + m_cluster.Unregister(this); + m_committed.Writer.TryComplete(); + return default; + } + + internal void Deliver(ReadOnlyMemory command) + { + m_committed.Writer.TryWrite(command); + } + + internal void SetLeadership(bool isLeader) + { + if (Volatile.Read(ref m_isLeader) == isLeader) + { + return; + } + Volatile.Write(ref m_isLeader, isLeader); + LeadershipChanged?.Invoke(isLeader); + } + + private void ThrowIfDisposed() + { + if (Volatile.Read(ref m_disposed) != 0) + { + throw new ObjectDisposedException(nameof(InProcessRaftConsensus)); + } + } + + private readonly InProcessRaftCluster m_cluster; + private readonly Channel> m_committed; + private int m_started; + private int m_disposed; + private bool m_isLeader; + } +} diff --git a/Libraries/Opc.Ua.Redundancy/Raft/RaftCsConsensus.cs b/Libraries/Opc.Ua.Redundancy/Raft/RaftCsConsensus.cs new file mode 100644 index 0000000000..aec2d28262 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/RaftCsConsensus.cs @@ -0,0 +1,348 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// =========================================================================== +// Adapter that binds the external Raft engine `RaftCs` +// (https://github.com/marcschier/raft-cs, shipped alongside the Crdt 1.1.0 +// libraries) to the in-repo IRaftConsensus seam. Opc.Ua.Redundancy references +// the `RaftCs` and `RaftCs.Transport` packages and defines OPCUA_RAFTCS, so +// this type is compiled into the assembly. The guard remains so the file can +// be excluded if the RaftCs dependency is ever removed. +// +// Usage: +// - Single-node / in-process: RaftCsConsensus.CreateSingleNode(). +// - Multi-pod: construct a RaftNode with durable storage +// (RaftCs.Storage.File) and a networked transport +// (RaftCs.Transport.NanoMsg, sharing the NanoMsg substrate with the CRDT +// gossip layer), then `new RaftCsConsensus(node)`. Wire either through +// RedundancyConsistencyOptions.RaftConsensusFactory (server) or the client +// AddRaftClientSharedStore/AddRedundantClientSharedStore factories. +// =========================================================================== + +#if OPCUA_RAFTCS +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Raft; +using Raft.Configuration; +using Raft.Storage; +using Raft.Transport; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: binds an external RaftCs replica to the + /// seam. Proposals map to , the committed apply + /// stream is , and leadership is observed from . + /// + public sealed class RaftCsConsensus : IRaftConsensus + { + /// + /// Creates an adapter over a RaftCs replica. + /// + /// The Raft replica (constructed with its storage and transport). + /// When true, the node is disposed with this adapter. + /// + /// How often the adapter samples to raise + /// (the node exposes state, not an event). + /// + /// + /// An optional resource (for example the in-memory network the transport belongs to) disposed after the node. + /// + /// + /// How long waits for the cluster to elect an initial leader before returning (so the + /// first proposals are not dropped during the initial election). Defaults to 10 seconds. + /// + public RaftCsConsensus( + RaftNode node, + bool ownsNode = true, + TimeSpan leadershipPollInterval = default, + IAsyncDisposable? ownedHost = null, + TimeSpan readyTimeout = default) + { + m_node = node ?? throw new ArgumentNullException(nameof(node)); + m_ownsNode = ownsNode; + m_ownedHost = ownedHost; + m_pollInterval = leadershipPollInterval <= TimeSpan.Zero + ? TimeSpan.FromMilliseconds(50) + : leadershipPollInterval; + m_readyTimeout = readyTimeout <= TimeSpan.Zero + ? TimeSpan.FromSeconds(10) + : readyTimeout; + } + + /// + /// Creates an adapter over a single-node RaftCs replica (in-memory storage and transport). The node + /// elects itself leader, so it provides a real, self-contained linearizable backend for single-process + /// deployments and tests; use for a multi-node cluster. + /// + /// This replica's unique, non-zero id. + /// How long to wait for self-election on start (defaults to 10 seconds). + public static RaftCsConsensus CreateSingleNode(ulong nodeId = 1, TimeSpan readyTimeout = default) + { + if (nodeId == 0) + { + throw new ArgumentOutOfRangeException(nameof(nodeId), "Raft node id must be non-zero."); + } + + // CA2000: ownership of the network/transport/node transfers to the + // returned adapter (ownsNode + ownedResources), which disposes them. +#pragma warning disable CA2000 + var network = new InMemoryNetwork(); + IRaftTransport transport = network.CreateNode(nodeId); + var storage = new MemoryStorage(new ConfState([nodeId])); + return CreateCluster( + nodeId, + transport, + storage, + new RaftNodeOptions { TickInterval = TimeSpan.FromMilliseconds(5) }, + config => config.ElectionTick = 3, + readyTimeout, + ownedResources: network); +#pragma warning restore CA2000 + } + + /// + /// Creates an adapter over a multi-node RaftCs replica with in-memory (volatile) storage. The + /// membership is the static set ; the caller supplies the network transport that + /// links the members. For durable storage (crash-safe WAL) use the overload that takes an + /// . + /// + /// This replica's unique, non-zero id (must be one of ). + /// The static cluster membership (voter ids). + /// This replica's transport bound to . + /// Optional driver options (tick interval, apply cap). + /// Optional callback to tune the (its Id is set). + /// How long waits for an initial leader. + public static RaftCsConsensus CreateCluster( + ulong nodeId, + ArrayOf memberIds, + IRaftTransport transport, + RaftNodeOptions? options = null, + Action? configure = null, + TimeSpan readyTimeout = default) + { + if (memberIds.IsEmpty) + { + throw new ArgumentException("At least one member id is required.", nameof(memberIds)); + } + + // CA2000: the storage is volatile (MemoryStorage is not disposable); + // the node/transport ownership transfers to the returned adapter. +#pragma warning disable CA2000 + var storage = new MemoryStorage(new ConfState(ToArray(memberIds))); + return CreateCluster(nodeId, transport, storage, options, configure, readyTimeout); +#pragma warning restore CA2000 + } + + /// + /// Creates an adapter over a multi-node RaftCs replica with caller-supplied durable storage (for + /// example a FileRaftStorage WAL). The membership is read from the storage's initial + /// ConfState. + /// + /// This replica's unique, non-zero id. + /// This replica's transport bound to . + /// The durable (or in-memory) Raft log/state storage carrying the membership. + /// Optional driver options (tick interval, apply cap). + /// Optional callback to tune the (its Id is set). + /// How long waits for an initial leader. + /// + /// An optional resource (for example the in-memory network, or the storage when it is disposable) disposed + /// after the node. + /// + public static RaftCsConsensus CreateCluster( + ulong nodeId, + IRaftTransport transport, + IRaftWritableStorage storage, + RaftNodeOptions? options = null, + Action? configure = null, + TimeSpan readyTimeout = default, + IAsyncDisposable? ownedResources = null) + { + if (nodeId == 0) + { + throw new ArgumentOutOfRangeException(nameof(nodeId), "Raft node id must be non-zero."); + } + if (transport == null) + { + throw new ArgumentNullException(nameof(transport)); + } + if (storage == null) + { + throw new ArgumentNullException(nameof(storage)); + } + + var config = new RaftConfig { Id = nodeId }; + configure?.Invoke(config); + + // CA2000: the node (which disposes the transport) and ownedResources + // transfer to the returned adapter, which disposes them. +#pragma warning disable CA2000 + var node = new RaftNode(config, storage, transport, options); + return new RaftCsConsensus(node, ownsNode: true, readyTimeout: readyTimeout, ownedHost: ownedResources); +#pragma warning restore CA2000 + } + + private static ulong[] ToArray(ArrayOf ids) + { + var result = new ulong[ids.Count]; + for (int ii = 0; ii < ids.Count; ii++) + { + result[ii] = ids[ii]; + } + return result; + } + + /// + public bool IsLeader => m_node.IsLeader; + + /// + public event Action? LeadershipChanged; + + /// + public ChannelReader> Committed => m_node.Committed; + + /// + public async ValueTask StartAsync(CancellationToken ct = default) + { + if (Interlocked.Exchange(ref m_started, 1) == 0) + { + await m_node.StartAsync(ct).ConfigureAwait(false); + m_leadershipLoop = Task.Run(() => WatchLeadershipAsync(m_cts.Token), m_cts.Token); + await WaitForInitialLeaderAsync(ct).ConfigureAwait(false); + } + } + + /// + public ValueTask ProposeAsync(ReadOnlyMemory command, CancellationToken ct = default) + { + // RaftNode.ProposeAsync is accepted on the leader; on a follower the + // node forwards the proposal to the leader it recognizes + // (RaftConfig.DisableProposalForwarding is false). The opaque command + // bytes — including this store's originator id and request id — are + // preserved, so the originating replica still correlates its own + // committed command back to the caller. + return m_node.ProposeAsync(command, ct); + } + + /// + public ValueTask CampaignAsync(CancellationToken ct = default) + { + return m_node.CampaignAsync(ct); + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref m_disposed, 1) != 0) + { + return; + } + + m_cts.Cancel(); + if (m_leadershipLoop != null) + { + try + { + await m_leadershipLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + + if (m_ownsNode) + { + await m_node.DisposeAsync().ConfigureAwait(false); + } + + if (m_ownedHost != null) + { + await m_ownedHost.DisposeAsync().ConfigureAwait(false); + } + + m_cts.Dispose(); + } + + private async Task WaitForInitialLeaderAsync(CancellationToken ct) + { + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeoutCts.CancelAfter(m_readyTimeout); + try + { + while (m_node.LeaderId == 0) + { + await Task.Delay(10, timeoutCts.Token).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // Timed out waiting for the initial election; proceed. Proposals + // are forwarded once a leader is elected, and each proposal is + // still bounded by its own cancellation token. + } + } + + private async Task WatchLeadershipAsync(CancellationToken ct) + { + bool last = false; + try + { + while (!ct.IsCancellationRequested) + { + bool now = m_node.IsLeader; + if (now != last) + { + last = now; + LeadershipChanged?.Invoke(now); + } + await Task.Delay(m_pollInterval, ct).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // shutdown + } + } + + private readonly RaftNode m_node; + private readonly bool m_ownsNode; + private readonly IAsyncDisposable? m_ownedHost; + private readonly TimeSpan m_pollInterval; + private readonly TimeSpan m_readyTimeout; + private readonly CancellationTokenSource m_cts = new(); + private Task? m_leadershipLoop; + private int m_started; + private int m_disposed; + } +} +#endif diff --git a/Libraries/Opc.Ua.Redundancy/Raft/RaftLeaderElection.cs b/Libraries/Opc.Ua.Redundancy/Raft/RaftLeaderElection.cs new file mode 100644 index 0000000000..898768f552 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/RaftLeaderElection.cs @@ -0,0 +1,116 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: an backed by native Raft leadership. Unlike + /// the lease-CAS based , leadership here is decided by the consensus + /// protocol itself (a single leader per term, no split-brain), and leadership transitions are pushed via + /// . + /// + public sealed class RaftLeaderElection : ILeaderElection + { + /// + /// Creates a Raft-backed leader election over the supplied consensus replica. + /// + /// The consensus replica whose leadership drives this election. + /// Optional logger. + public RaftLeaderElection(IRaftConsensus consensus, ILogger? logger = null) + { + m_consensus = consensus ?? throw new ArgumentNullException(nameof(consensus)); + m_logger = logger; + m_consensus.LeadershipChanged += OnLeadershipChanged; + } + + /// + public bool IsLeader => m_consensus.IsLeader; + + /// + public event Action? LeadershipChanged; + + /// + public async ValueTask TryAcquireOrRenewAsync(CancellationToken ct = default) + { + await m_consensus.CampaignAsync(ct).ConfigureAwait(false); + return m_consensus.IsLeader; + } + + /// + public void Start() + { + // Raft leadership is event-driven (no lease-renew loop). Starting + // simply ensures the underlying consensus replica is running; the + // ILeaderElection contract is fire-and-forget (void), so failures + // are logged rather than surfaced. + if (Interlocked.Exchange(ref m_started, 1) == 0) + { + _ = StartConsensusAsync(); + } + } + + /// + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref m_disposed, 1) != 0) + { + return default; + } + m_consensus.LeadershipChanged -= OnLeadershipChanged; + return default; + } + + private async Task StartConsensusAsync() + { + try + { + await m_consensus.StartAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + m_logger?.LogError(ex, "Raft consensus replica failed to start for leader election."); + } + } + + private void OnLeadershipChanged(bool value) + { + LeadershipChanged?.Invoke(value); + } + + private readonly IRaftConsensus m_consensus; + private readonly ILogger? m_logger; + private int m_started; + private int m_disposed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy/Raft/RaftSharedKeyValueStore.cs b/Libraries/Opc.Ua.Redundancy/Raft/RaftSharedKeyValueStore.cs new file mode 100644 index 0000000000..7cae89a9cf --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/RaftSharedKeyValueStore.cs @@ -0,0 +1,540 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: a strongly-consistent (CP), linearizable + /// implemented as a replicated state machine over an log. Unlike the + /// eventually-consistent , this store provides a real + /// and a change-feed, which makes it the right backend + /// for exactly-once primitives (the single-use session nonce registry) and master-write lease/election. + /// + /// + /// + /// Every mutation (Set/Delete/CAS) is encoded as an opaque command and + /// d. A single applier drains in + /// log order and mutates a materialized map deterministically — so a compare-and-swap is decided at apply time + /// against the committed state, identically on every replica. The originating store correlates the applied command + /// back to the caller (by request id) to return the result. + /// + /// + /// Reads (/) are served from the materialized map. The change-feed + /// () is derived from the same apply stream, so watchers observe changes in commit order. + /// + /// + public sealed class RaftSharedKeyValueStore : ISharedKeyValueStore, IAsyncDisposable + { + /// + /// Creates a store backed by a private single-node in-process consensus replica (always the leader). Useful + /// for single-process deployments and tests. + /// + // CA2000: ownership of the created replica transfers to this store + // (ownsConsensus: true) and it is disposed in DisposeAsync. +#pragma warning disable CA2000 + public RaftSharedKeyValueStore() + : this(new InProcessRaftConsensus(), ownsConsensus: true) + { + } +#pragma warning restore CA2000 + + /// + /// Creates a store backed by the supplied consensus replica. + /// + /// The consensus replica that replicates this store's commands. + /// + /// When true, the consensus replica is disposed together with this store. Pass false when the + /// replica is shared (for example with a ) and owned elsewhere. + /// + /// + /// How long a proposal waits to commit before failing with a , so a caller is + /// never blocked indefinitely when there is no leader, a leadership change discards the entry, or quorum is + /// lost. Defaults to 30 seconds; pass to wait only on the caller's + /// token. + /// + public RaftSharedKeyValueStore( + IRaftConsensus consensus, + bool ownsConsensus = false, + TimeSpan commitTimeout = default) + { + m_consensus = consensus ?? throw new ArgumentNullException(nameof(consensus)); + m_ownsConsensus = ownsConsensus; + m_commitTimeout = commitTimeout == TimeSpan.Zero ? TimeSpan.FromSeconds(30) : commitTimeout; + } + + /// + public async ValueTask<(bool Found, ByteString Value)> TryGetAsync(string key, CancellationToken ct = default) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + await EnsureStartedAsync(ct).ConfigureAwait(false); + lock (m_lock) + { + return m_state.TryGetValue(key, out ByteString value) ? (true, value) : (false, default); + } + } + + /// + public async ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) + { + await ProposeAsync(OpSet, key, default, value, ct).ConfigureAwait(false); + } + + /// + public ValueTask CompareAndSwapAsync( + string key, + ByteString expected, + ByteString value, + CancellationToken ct = default) + { + return ProposeAsync(OpCas, key, expected, value, ct); + } + + /// + public ValueTask DeleteAsync(string key, CancellationToken ct = default) + { + return ProposeAsync(OpDelete, key, default, default, ct); + } + + /// + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + keyPrefix ??= string.Empty; + await EnsureStartedAsync(ct).ConfigureAwait(false); + + List> snapshot; + lock (m_lock) + { + snapshot = new List>(m_state.Count); + foreach (KeyValuePair entry in m_state) + { + if (entry.Key.StartsWith(keyPrefix, StringComparison.Ordinal)) + { + snapshot.Add(entry); + } + } + } + + foreach (KeyValuePair entry in snapshot) + { + ct.ThrowIfCancellationRequested(); + yield return entry; + } + } + + /// + public async IAsyncEnumerable WatchAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + await EnsureStartedAsync(ct).ConfigureAwait(false); + + var watcher = new Watcher(keyPrefix ?? string.Empty); + lock (m_lock) + { + m_watchers.Add(watcher); + } + + try + { + await foreach (KeyValueChange change in watcher.Channel.Reader + .ReadAllAsync(ct) + .ConfigureAwait(false)) + { + yield return change; + } + } + finally + { + lock (m_lock) + { + m_watchers.Remove(watcher); + } + watcher.Channel.Writer.TryComplete(); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref m_disposed, 1) != 0) + { + return; + } + + m_cts.Cancel(); + if (m_applyLoop != null) + { + try + { + await m_applyLoop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + + lock (m_lock) + { + foreach (Watcher watcher in m_watchers) + { + watcher.Channel.Writer.TryComplete(); + } + m_watchers.Clear(); + } + + foreach (KeyValuePair> pending in m_pending) + { + pending.Value.TrySetCanceled(); + } + m_pending.Clear(); + + if (m_ownsConsensus) + { + await m_consensus.DisposeAsync().ConfigureAwait(false); + } + + m_cts.Dispose(); + m_startGate.Dispose(); + } + + private async ValueTask ProposeAsync( + byte op, + string key, + ByteString expected, + ByteString value, + CancellationToken ct) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + await EnsureStartedAsync(ct).ConfigureAwait(false); + + long requestId = Interlocked.Increment(ref m_requestId); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + m_pending[requestId] = tcs; + + // Bound the wait for commit: either the caller's token or the commit + // timeout fails the pending proposal, so a no-leader / leadership- + // change / lost-quorum window never blocks the caller indefinitely. + using var commitCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + if (m_commitTimeout != Timeout.InfiniteTimeSpan) + { + commitCts.CancelAfter(m_commitTimeout); + } + using CancellationTokenRegistration reg = commitCts.Token.Register(() => + { + if (m_pending.TryRemove(requestId, out TaskCompletionSource? pending)) + { + if (ct.IsCancellationRequested) + { + pending.TrySetCanceled(ct); + } + else + { + pending.TrySetException(new TimeoutException( + "The Raft proposal did not commit within the commit timeout (no leader, leadership " + + "change, or lost quorum). Retry the operation.")); + } + } + }); + + byte[] command = Encode(op, m_originator, requestId, key, expected, value); + try + { + await m_consensus.ProposeAsync(command, ct).ConfigureAwait(false); + } + catch + { + m_pending.TryRemove(requestId, out _); + throw; + } + + return await tcs.Task.ConfigureAwait(false); + } + + private async ValueTask EnsureStartedAsync(CancellationToken ct) + { + if (Volatile.Read(ref m_started) == 1) + { + return; + } + + await m_startGate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (m_started == 0) + { + await m_consensus.StartAsync(ct).ConfigureAwait(false); + m_applyLoop = Task.Run(() => ApplyLoopAsync(m_cts.Token), m_cts.Token); + Volatile.Write(ref m_started, 1); + } + } + finally + { + m_startGate.Release(); + } + } + + private async Task ApplyLoopAsync(CancellationToken ct) + { + try + { + await foreach (ReadOnlyMemory command in m_consensus.Committed + .ReadAllAsync(ct) + .ConfigureAwait(false)) + { + Apply(command.Span); + } + } + catch (OperationCanceledException) + { + // shutdown + } + catch (Exception ex) + { + // The applier is the sole completer of pending proposals; if it + // ever dies unexpectedly, fail every pending proposal so callers + // do not hang. + FailAllPending(ex); + } + } + + private void FailAllPending(Exception error) + { + foreach (KeyValuePair> pending in m_pending) + { + pending.Value.TrySetException(error); + } + m_pending.Clear(); + } + + private void Apply(ReadOnlySpan command) + { + byte op = command[0]; + var originator = new Guid(command.Slice(1, 16).ToArray()); + long requestId = BinaryPrimitives.ReadInt64LittleEndian(command.Slice(17, 8)); + int offset = 25; + + int keyLength = BinaryPrimitives.ReadInt32LittleEndian(command.Slice(offset, 4)); + offset += 4; + string key = Encoding.UTF8.GetString(command.Slice(offset, keyLength).ToArray()); + offset += keyLength; + + ByteString expected = default; + if (op == OpCas) + { + expected = ReadValue(command, ref offset); + } + + ByteString value = default; + if (op != OpDelete) + { + value = ReadValue(command, ref offset); + } + + bool result = false; + KeyValueChange? change = null; + lock (m_lock) + { + switch (op) + { + case OpSet: + m_state[key] = value; + change = new KeyValueChange { Kind = KeyValueChangeKind.Set, Key = key, Value = value }; + result = true; + break; + case OpDelete: + result = m_state.Remove(key); + if (result) + { + change = new KeyValueChange { Kind = KeyValueChangeKind.Delete, Key = key }; + } + break; + case OpCas: + bool present = m_state.TryGetValue(key, out ByteString current); + bool matches = expected.IsNull ? !present : present && current.Equals(expected); + if (matches) + { + m_state[key] = value; + change = new KeyValueChange { Kind = KeyValueChangeKind.Set, Key = key, Value = value }; + result = true; + } + break; + } + + if (change != null) + { + PublishLocked(change); + } + } + + if (originator == m_originator && m_pending.TryRemove(requestId, out TaskCompletionSource? tcs)) + { + tcs.TrySetResult(result); + } + } + + private void PublishLocked(KeyValueChange change) + { + for (int ii = 0; ii < m_watchers.Count; ii++) + { + Watcher watcher = m_watchers[ii]; + if (change.Key.StartsWith(watcher.Prefix, StringComparison.Ordinal)) + { + watcher.Channel.Writer.TryWrite(change); + } + } + } + + private static byte[] Encode( + byte op, + Guid originator, + long requestId, + string key, + ByteString expected, + ByteString value) + { + byte[] keyBytes = Encoding.UTF8.GetBytes(key); + byte[]? expectedBytes = op == OpCas && !expected.IsNull ? expected.ToArray() : null; + byte[]? valueBytes = op != OpDelete && !value.IsNull ? value.ToArray() : null; + + int length = 1 + 16 + 8 + 4 + keyBytes.Length; + if (op == OpCas) + { + length += 4 + (expectedBytes?.Length ?? 0); + } + if (op != OpDelete) + { + length += 4 + (valueBytes?.Length ?? 0); + } + + byte[] buffer = new byte[length]; + int offset = 0; + buffer[offset++] = op; + originator.ToByteArray().CopyTo(buffer, offset); + offset += 16; + BinaryPrimitives.WriteInt64LittleEndian(buffer.AsSpan(offset, 8), requestId); + offset += 8; + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(offset, 4), keyBytes.Length); + offset += 4; + keyBytes.CopyTo(buffer, offset); + offset += keyBytes.Length; + + if (op == OpCas) + { + WriteValue(buffer, ref offset, expected.IsNull, expectedBytes); + } + if (op != OpDelete) + { + WriteValue(buffer, ref offset, value.IsNull, valueBytes); + } + + return buffer; + } + + private static void WriteValue(byte[] buffer, ref int offset, bool isNull, byte[]? bytes) + { + int len = isNull ? -1 : bytes?.Length ?? 0; + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(offset, 4), len); + offset += 4; + if (len > 0 && bytes != null) + { + bytes.CopyTo(buffer, offset); + offset += bytes.Length; + } + } + + private static ByteString ReadValue(ReadOnlySpan command, ref int offset) + { + int len = BinaryPrimitives.ReadInt32LittleEndian(command.Slice(offset, 4)); + offset += 4; + if (len < 0) + { + return default; + } + if (len == 0) + { + return new ByteString(Array.Empty()); + } + byte[] bytes = command.Slice(offset, len).ToArray(); + offset += len; + return new ByteString(bytes); + } + + private sealed class Watcher + { + public Watcher(string prefix) + { + Prefix = prefix; + } + + public string Prefix { get; } + + public Channel Channel { get; } = + System.Threading.Channels.Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + } + + private const byte OpSet = 0; + private const byte OpDelete = 1; + private const byte OpCas = 2; + + private readonly IRaftConsensus m_consensus; + private readonly bool m_ownsConsensus; + private readonly TimeSpan m_commitTimeout; + private readonly Guid m_originator = Guid.NewGuid(); + private readonly Dictionary m_state = new(StringComparer.Ordinal); + private readonly List m_watchers = []; + private readonly ConcurrentDictionary> m_pending = new(); + private readonly Lock m_lock = new(); + private readonly SemaphoreSlim m_startGate = new(1, 1); + private readonly CancellationTokenSource m_cts = new(); + private Task? m_applyLoop; + private long m_requestId; + private int m_started; + private int m_disposed; + } +} diff --git a/Libraries/Opc.Ua.Redundancy/Raft/RedundancyConsistencyMode.cs b/Libraries/Opc.Ua.Redundancy/Raft/RedundancyConsistencyMode.cs new file mode 100644 index 0000000000..718c5df134 --- /dev/null +++ b/Libraries/Opc.Ua.Redundancy/Raft/RedundancyConsistencyMode.cs @@ -0,0 +1,52 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: selects how a RedundantServerSet replicates shared state. + /// + public enum RedundancyConsistencyMode + { + /// + /// Eventual consistency (AP): bulk replicated state is served by the leaderless, gossip-based + /// and is complemented by a strongly-consistent Raft layer for the + /// linearizable primitives that need it (compare-and-swap, change-feed, single-use nonces, lease/leader + /// election). This is the default and preserves today's behaviour and performance for the common case. + /// + Eventual, + + /// + /// Strong consistency (CP): all shared state is served by the linearizable + /// ; no CRDT is used. Choose this when every replicated value must be + /// linearizable at the cost of requiring a quorum for writes. + /// + Strong + } +} diff --git a/Libraries/Opc.Ua.Server/Diagnostics/AuditEvents.cs b/Libraries/Opc.Ua.Server/Diagnostics/AuditEvents.cs index 78aa298064..649b105be5 100644 --- a/Libraries/Opc.Ua.Server/Diagnostics/AuditEvents.cs +++ b/Libraries/Opc.Ua.Server/Diagnostics/AuditEvents.cs @@ -1235,6 +1235,60 @@ public static void ReportAuditCloseSessionEvent( } } + /// + /// Reports an audit event when a session is restored (materialized) from + /// a shared store on a standby replica during a high-availability + /// failover. This is a security-relevant provenance event distinct from + /// the regular activate-session audit. + /// + /// The server which reports audit events. + /// + /// The audit entry id (a one-way token digest is used for provenance so + /// the raw authentication token is not exposed). + /// + /// The restored session. + /// A contextual logger to log to. + /// The source name of the audit event. + public static void ReportAuditSessionRestoredEvent( + this IAuditEventServer? server, + string auditEntryId, + ISession session, + ILogger logger, + string sourceName = "Session/RestoredFromSharedStore") + { + if (server?.Auditing != true) + { + // current server does not support auditing + return; + } + + try + { + ISystemContext systemContext = server.DefaultAuditContext; + + // raise an audit event. + var e = new AuditSessionEventState(null); + + var message = new TranslationInfo( + "AuditSessionRestoredEvent", + "en-US", + $"Session with ID:{session?.Id} was restored from the shared store on this replica."); + + InitializeAuditSessionEvent(systemContext, e, message, true, session!, auditEntryId); + + e.SetChildValue(systemContext, BrowseNames.SourceName, sourceName, false); + + server.ReportAuditEvent(systemContext, e); + } + catch (Exception ex) + { + logger.LogError( + ex, + "Error while reporting AuditSessionEventState restored event for SessionId {SessionId}.", + session?.Id); + } + } + /// /// Reports an audit session event for the transfer subscription. /// diff --git a/Libraries/Opc.Ua.Server/Diagnostics/DiagnosticsNodeManager.cs b/Libraries/Opc.Ua.Server/Diagnostics/DiagnosticsNodeManager.cs index 3660678269..9cb772b092 100644 --- a/Libraries/Opc.Ua.Server/Diagnostics/DiagnosticsNodeManager.cs +++ b/Libraries/Opc.Ua.Server/Diagnostics/DiagnosticsNodeManager.cs @@ -372,6 +372,29 @@ protected virtual void AddSdkImplementedOptionalChildren( } } + /// + /// Applies the OPC 10000-5 subtype model for Server.ServerRedundancy. + /// + /// The ServerRedundancy instance. + private static void ApplyServerRedundancyTypeDefinition(ServerRedundancyState serverRedundancy) + { + if (serverRedundancy == null) + { + throw new ArgumentNullException(nameof(serverRedundancy)); + } + + RedundancySupport mode = serverRedundancy.RedundancySupport?.Value ?? RedundancySupport.None; + serverRedundancy.TypeDefinitionId = mode switch + { + RedundancySupport.Transparent => TransparentRedundancyTypeId, + RedundancySupport.Cold or + RedundancySupport.Warm or + RedundancySupport.Hot or + RedundancySupport.HotAndMirrored => NonTransparentRedundancyTypeId, + _ => ServerRedundancyTypeId + }; + } + private void AddServerSdkOptionalChildren( ISystemContext context, ServerObjectState serverObject) @@ -385,6 +408,7 @@ private void AddServerSdkOptionalChildren( .AddNamespaces(context) .AddUrisVersion(context) .AddEstimatedReturnTime(context) + .AddRequestServerStateChange(context) .AddLocalTime(context); if (serverObject.ServerCapabilities != null) @@ -394,6 +418,7 @@ private void AddServerSdkOptionalChildren( } if (serverObject.ServerRedundancy != null) { + ApplyServerRedundancyTypeDefinition(serverObject.ServerRedundancy); AddServerRedundancySdkOptionalChildren( context, serverObject.ServerRedundancy); } @@ -438,6 +463,74 @@ private static void AddServerRedundancySdkOptionalChildren( ServerRedundancyState serverRedundancy) { serverRedundancy.AddRedundantServerArray(context); + AddServerRedundancyStringProperty( + context, + serverRedundancy, + BrowseNames.CurrentServerId, + VariableIds.Server_ServerRedundancy_CurrentServerId, + ValueRanks.Scalar); + AddServerRedundancyStringArrayProperty( + context, + serverRedundancy, + BrowseNames.ServerUriArray, + VariableIds.Server_ServerRedundancy_ServerUriArray, + ValueRanks.OneDimension); + } + + private static void AddServerRedundancyStringProperty( + ISystemContext context, + ServerRedundancyState serverRedundancy, + string browseName, + NodeId nodeId, + int valueRank) + { + if (serverRedundancy.FindChild(context, new QualifiedName(browseName, 0)) != null) + { + return; + } + + var property = new PropertyState.Implementation(serverRedundancy) + { + SymbolicName = browseName, + NodeId = nodeId, + BrowseName = new QualifiedName(browseName, 0), + DisplayName = new LocalizedText(browseName), + DataType = DataTypeIds.String, + ValueRank = valueRank, + AccessLevel = AccessLevels.CurrentRead, + UserAccessLevel = AccessLevels.CurrentRead, + ReferenceTypeId = ReferenceTypeIds.HasProperty, + TypeDefinitionId = VariableTypeIds.PropertyType + }; + serverRedundancy.AddChild(property); + } + + private static void AddServerRedundancyStringArrayProperty( + ISystemContext context, + ServerRedundancyState serverRedundancy, + string browseName, + NodeId nodeId, + int valueRank) + { + if (serverRedundancy.FindChild(context, new QualifiedName(browseName, 0)) != null) + { + return; + } + + var property = new PropertyState>.Implementation(serverRedundancy) + { + SymbolicName = browseName, + NodeId = nodeId, + BrowseName = new QualifiedName(browseName, 0), + DisplayName = new LocalizedText(browseName), + DataType = DataTypeIds.String, + ValueRank = valueRank, + AccessLevel = AccessLevels.CurrentRead, + UserAccessLevel = AccessLevels.CurrentRead, + ReferenceTypeId = ReferenceTypeIds.HasProperty, + TypeDefinitionId = VariableTypeIds.PropertyType + }; + serverRedundancy.AddChild(property); } private static void AddHistoryCapabilitiesSdkOptionalChildren( @@ -2333,6 +2426,10 @@ private void DoSample(object? state) }; } + private static readonly NodeId ServerRedundancyTypeId = new(2034); + private static readonly NodeId TransparentRedundancyTypeId = new(2036); + private static readonly NodeId NonTransparentRedundancyTypeId = new(2039); + private static readonly NodeId[] s_kWellKnownRoles = [ ObjectIds.WellKnownRole_Anonymous, diff --git a/Libraries/Opc.Ua.Server/Fluent/EventPublishOptions.cs b/Libraries/Opc.Ua.Server/Fluent/EventPublishOptions.cs index 8bb7d270d5..14f2ef190c 100644 --- a/Libraries/Opc.Ua.Server/Fluent/EventPublishOptions.cs +++ b/Libraries/Opc.Ua.Server/Fluent/EventPublishOptions.cs @@ -70,6 +70,13 @@ public sealed record EventPublishOptions /// public bool SkipDefaultPopulation { get; init; } + /// + /// Optional provider used to populate EventId when the event did + /// not set one explicitly. Default null keeps the current random + /// UUID behavior. + /// + public IEventIdProvider? EventIdProvider { get; init; } + /// /// When true, the notifier node is added to the manager's /// root-notifier collection so its events propagate to clients that diff --git a/Libraries/Opc.Ua.Server/Fluent/EventSourceRegistry.cs b/Libraries/Opc.Ua.Server/Fluent/EventSourceRegistry.cs index 64d1197385..864a7f73de 100644 --- a/Libraries/Opc.Ua.Server/Fluent/EventSourceRegistry.cs +++ b/Libraries/Opc.Ua.Server/Fluent/EventSourceRegistry.cs @@ -523,7 +523,7 @@ private void DispatchEvent(SourceEntry entry, ISystemContext context, BaseEventS { if (!entry.Options.SkipDefaultPopulation) { - PopulateDefaults(entry.Notifier, context, e); + PopulateDefaults(entry.Notifier, context, e, entry.Options.EventIdProvider); } entry.Notifier.ReportEvent(context, e); } @@ -552,15 +552,12 @@ private void DispatchEvent(SourceEntry entry, ISystemContext context, BaseEventS /// ReceiveTime, Severity (Medium when 0), /// Message. /// - private static void PopulateDefaults(BaseObjectState notifier, ISystemContext context, BaseEventState e) + private static void PopulateDefaults( + BaseObjectState notifier, + ISystemContext context, + BaseEventState e, + IEventIdProvider? eventIdProvider) { - if (e.EventId == null || e.EventId.Value.IsNull) - { - e.EventId = PropertyState.With( - e, - Uuid.NewUuid().ToByteString()); - } - if (e.EventType == null || e.EventType.Value.IsNull) { NodeId defaultType = e.GetDefaultTypeDefinitionId(context); @@ -590,7 +587,6 @@ private static void PopulateDefaults(BaseObjectState notifier, ISystemContext co } if (e.Time == null || e.Time.Value.IsNull) - { e.Time = PropertyState.With(e, DateTimeUtc.Now); } @@ -611,6 +607,13 @@ private static void PopulateDefaults(BaseObjectState notifier, ISystemContext co e.Message ??= PropertyState.With( e, new LocalizedText(string.Empty)); + + if (e.EventId == null || e.EventId.Value.IsNull) + { + e.EventId = PropertyState.With( + e, + eventIdProvider?.CreateEventId(notifier, context, e) ?? Uuid.NewUuid().ToByteString()); + } } private void ThrowIfDisposed() diff --git a/Libraries/Opc.Ua.Server/Fluent/IEventIdProvider.cs b/Libraries/Opc.Ua.Server/Fluent/IEventIdProvider.cs new file mode 100644 index 0000000000..1e8d05c6b9 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Fluent/IEventIdProvider.cs @@ -0,0 +1,50 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Server.Fluent +{ + /// + /// Supplies event ids for fluent event publishing. + /// + /// + /// When no provider is configured, fluent event publishing keeps the existing random UUID EventId behavior. + /// Distributed deployments can configure a provider that derives stable ids from replica-shared event identity. + /// + public interface IEventIdProvider + { + /// + /// Creates an event id for an event whose EventId field was not already populated. + /// + /// The notifier that reports the event. + /// The system context. + /// The event being reported. + /// The event id. + ByteString CreateEventId(BaseObjectState notifier, ISystemContext context, BaseEventState eventState); + } +} diff --git a/Libraries/Opc.Ua.Server/Hosting/IServerStartupTask.cs b/Libraries/Opc.Ua.Server/Hosting/IServerStartupTask.cs new file mode 100644 index 0000000000..5897d4c4f6 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Hosting/IServerStartupTask.cs @@ -0,0 +1,53 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server.Hosting +{ + /// + /// A task run by the hosted server immediately after the server has + /// started, with access to the live . Every + /// implementation registered in DI is invoked once. This is the seam + /// features use to wire runtime behavior that needs the fully-initialized + /// server (loaded node managers, the populated message context, the + /// ServerObject, the session manager) without subclassing + /// . + /// + public interface IServerStartupTask + { + /// + /// Invoked once after the server has started. + /// + /// The live server internals. + /// Cancellation token. + ValueTask OnServerStartedAsync(IServerInternal server, CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Server/Hosting/OpcUaServerHostedService.cs b/Libraries/Opc.Ua.Server/Hosting/OpcUaServerHostedService.cs index 018aab44bc..de4c52f8a7 100644 --- a/Libraries/Opc.Ua.Server/Hosting/OpcUaServerHostedService.cs +++ b/Libraries/Opc.Ua.Server/Hosting/OpcUaServerHostedService.cs @@ -32,6 +32,7 @@ using System.IO; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -216,6 +217,9 @@ await securityOptions } m_server = new StandardServer(m_telemetry, m_timeProvider); + m_server.SessionManagerFactory = m_services.GetService(); + m_server.RedundantServerSetProvider = m_services.GetService(); + m_server.GetEndpointsDirector = m_services.GetService(); foreach (OpcUaServerNodeManagerRegistration reg in m_registrations) { if (reg.AsyncFactory is not null) @@ -233,6 +237,16 @@ await securityOptions RegisterIdentityAuthenticators(); RegisterIdentityAugmenters(); + // Run post-start tasks (e.g. distributed address-space wiring) + // now that the server is fully initialized and CurrentInstance is + // available. Features register these without subclassing the server. + foreach (IServerStartupTask startupTask in m_services.GetServices()) + { + await startupTask + .OnServerStartedAsync(m_server.CurrentInstance, stoppingToken) + .ConfigureAwait(false); + } + foreach (string url in urls) { m_logger.LogInformation("OPC UA server listening at {Endpoint}.", url); diff --git a/Libraries/Opc.Ua.Server/Identity/Authenticators/KeyCredentialBridgeAuthenticator.cs b/Libraries/Opc.Ua.Server/Identity/Authenticators/KeyCredentialBridgeAuthenticator.cs index 4b14e92ea5..15ee0d1cfb 100644 --- a/Libraries/Opc.Ua.Server/Identity/Authenticators/KeyCredentialBridgeAuthenticator.cs +++ b/Libraries/Opc.Ua.Server/Identity/Authenticators/KeyCredentialBridgeAuthenticator.cs @@ -155,7 +155,7 @@ public async ValueTask AuthenticateAsync( token.Nonce, token.IssuedAt); - if (!FixedTimeEquals(suppliedProof, expectedProof)) + if (!CryptoUtils.FixedTimeEquals(suppliedProof, expectedProof)) { return Reject(StatusCodes.BadIdentityTokenRejected, "KeyCredential proof validation failed."); } @@ -299,21 +299,6 @@ private static byte[] ComputeProof( return hmac.ComputeHash(Encoding.UTF8.GetBytes(input)); } - private static bool FixedTimeEquals(byte[] left, byte[] right) - { - if (left.Length != right.Length) - { - return false; - } - - int difference = 0; - for (int i = 0; i < left.Length; i++) - { - difference |= left[i] ^ right[i]; - } - return difference == 0; - } - private static string GetString(JsonElement root, string propertyName) { if (!root.TryGetProperty(propertyName, out JsonElement value) || diff --git a/Libraries/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs b/Libraries/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs index a665242e76..c290969760 100644 --- a/Libraries/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs +++ b/Libraries/Opc.Ua.Server/NodeManager/AsyncCustomNodeManager.cs @@ -52,7 +52,8 @@ namespace Opc.Ua.Server public class AsyncCustomNodeManager : IAsyncNodeManager, INodeIdFactory, - IDisposable + IDisposable, + ILocalAddressSpaceSource { /// /// Initializes the node manager. @@ -243,6 +244,16 @@ public virtual NodeId New(ISystemContext context, NodeState node) return node.NodeId; } + /// + ILocalAddressSpace ILocalAddressSpaceSource.CreateLocalAddressSpace() + { + return new PredefinedNodesAddressSpace( + SystemContext, + PredefinedNodes, + (node, cancellationToken) => AddPredefinedNodeAsync(SystemContext, node, cancellationToken), + (nodeId, cancellationToken) => DeleteNodeAsync(SystemContext, nodeId, cancellationToken)); + } + /// /// Gets the server that the node manager belongs to. /// diff --git a/Libraries/Opc.Ua.Server/NodeManager/ContinuationPoint.cs b/Libraries/Opc.Ua.Server/NodeManager/ContinuationPoint.cs index 0e6678795e..9dddd2aec4 100644 --- a/Libraries/Opc.Ua.Server/NodeManager/ContinuationPoint.cs +++ b/Libraries/Opc.Ua.Server/NodeManager/ContinuationPoint.cs @@ -84,6 +84,11 @@ protected virtual void Dispose(bool disposing) /// public object NodeToBrowse { get; set; } = null!; + /// + /// The NodeId from the original Browse request, when available. + /// + public NodeId RequestedNodeId { get; set; } + /// /// The maximum number of results to return. /// diff --git a/Libraries/Opc.Ua.Server/NodeManager/CustomNodeManager.cs b/Libraries/Opc.Ua.Server/NodeManager/CustomNodeManager.cs index 96749638fe..36a2f533c1 100644 --- a/Libraries/Opc.Ua.Server/NodeManager/CustomNodeManager.cs +++ b/Libraries/Opc.Ua.Server/NodeManager/CustomNodeManager.cs @@ -30,6 +30,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; using System.Threading; @@ -49,7 +50,8 @@ namespace Opc.Ua.Server /// is not part of the SDK because most real implementations of a INodeManager will need to /// modify the behavior of the base class. /// - public partial class CustomNodeManager2 : INodeManager3, INodeIdFactory, IDisposable + public partial class CustomNodeManager2 : INodeManager3, INodeIdFactory, IDisposable, + ILocalAddressSpaceSource { /// /// Initializes the node manager. @@ -211,6 +213,20 @@ public virtual NodeId New(ISystemContext context, NodeState node) return node.NodeId; } + /// + ILocalAddressSpace ILocalAddressSpaceSource.CreateLocalAddressSpace() + { + return new PredefinedNodesAddressSpace( + SystemContext, + PredefinedNodes, + (node, cancellationToken) => + { + AddPredefinedNode(SystemContext, node); + return default; + }, + (nodeId, cancellationToken) => new ValueTask(DeleteNode(SystemContext, nodeId))); + } + /// /// Acquires the lock on the node manager. /// diff --git a/Libraries/Opc.Ua.Server/NodeManager/ILocalAddressSpace.cs b/Libraries/Opc.Ua.Server/NodeManager/ILocalAddressSpace.cs new file mode 100644 index 0000000000..1a6059b5f6 --- /dev/null +++ b/Libraries/Opc.Ua.Server/NodeManager/ILocalAddressSpace.cs @@ -0,0 +1,90 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server +{ + /// + /// Abstraction over the in-process node graph that a distributed + /// address-space synchronizer reads from (outbound capture) and writes to + /// (inbound apply). A node manager adapts its PredefinedNodes + /// dictionary to this interface; tests use a dictionary-backed + /// implementation. + /// + public interface ILocalAddressSpace + { + /// + /// The system context used to (de)serialize nodes. + /// + ISystemContext Context { get; } + + /// + /// A snapshot of the current top-level nodes. + /// + IEnumerable Nodes { get; } + + /// + /// Raised after a node is added to the address space. + /// + event Action? NodeAdded; + + /// + /// Raised after a node is removed from the address space. + /// + event Action? NodeRemoved; + + /// + /// Looks up a node by identifier. + /// + /// The node identifier. + /// The resolved node when found. + /// true when the node exists. + bool TryGetNode(NodeId nodeId, [NotNullWhen(true)] out NodeState? node); + + /// + /// Adds or replaces a node, raising . + /// + /// The node to add or replace. + /// The cancellation token. + ValueTask AddOrUpdateNodeAsync(NodeState node, CancellationToken cancellationToken = default); + + /// + /// Removes a node, raising when present. + /// + /// The node identifier. + /// The cancellation token. + /// true when a node was removed. + ValueTask RemoveNodeAsync(NodeId nodeId, CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Server/NodeManager/ILocalAddressSpaceSource.cs b/Libraries/Opc.Ua.Server/NodeManager/ILocalAddressSpaceSource.cs new file mode 100644 index 0000000000..2952370902 --- /dev/null +++ b/Libraries/Opc.Ua.Server/NodeManager/ILocalAddressSpaceSource.cs @@ -0,0 +1,44 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Server +{ + /// + /// Opt-in interface implemented by node managers that expose their local + /// address space to a distributed address-space synchronizer. + /// + public interface ILocalAddressSpaceSource + { + /// + /// Creates an adapter over the node manager's local address space. + /// + /// The local address space adapter. + ILocalAddressSpace CreateLocalAddressSpace(); + } +} diff --git a/Libraries/Opc.Ua.Server/NodeManager/MasterNodeManager.cs b/Libraries/Opc.Ua.Server/NodeManager/MasterNodeManager.cs index ef23ab6bb8..09023c74ec 100644 --- a/Libraries/Opc.Ua.Server/NodeManager/MasterNodeManager.cs +++ b/Libraries/Opc.Ua.Server/NodeManager/MasterNodeManager.cs @@ -1317,7 +1317,14 @@ private async ValueTask DispatchDeleteReferenceAsync( return null; } - /// + /// + /// Registers a set of node ids. + /// + /// + /// The default master node manager returns the requested node ids unchanged, so registered-node + /// results remain stable across replicas without additional mirroring. + /// + /// is null. public virtual void RegisterNodes( OperationContext context, ArrayOf nodesToRegister, @@ -2162,6 +2169,7 @@ protected async ValueTask BrowseAsync( Manager = nodeManager!, View = view, NodeToBrowse = handle, + RequestedNodeId = nodeToBrowse.NodeId, MaxResultsToReturn = maxReferencesPerNode, BrowseDirection = nodeToBrowse.BrowseDirection, ReferenceTypeId = nodeToBrowse.ReferenceTypeId, diff --git a/Libraries/Opc.Ua.Server/NodeManager/PredefinedNodesAddressSpace.cs b/Libraries/Opc.Ua.Server/NodeManager/PredefinedNodesAddressSpace.cs new file mode 100644 index 0000000000..ee48ebf75f --- /dev/null +++ b/Libraries/Opc.Ua.Server/NodeManager/PredefinedNodesAddressSpace.cs @@ -0,0 +1,136 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server +{ + /// + /// Shared adapter over a node manager's + /// PredefinedNodes dictionary. The snapshot/lookup behaviour is + /// identical for the synchronous and asynchronous node manager base + /// classes; the add and remove operations are supplied as delegates so the + /// adapter can drive either the synchronous or the asynchronous predefined + /// node pipeline without duplicating the boilerplate. + /// + internal sealed class PredefinedNodesAddressSpace : ILocalAddressSpace + { + /// + /// Creates the adapter. + /// + /// The system context used to (de)serialize nodes. + /// The node manager's predefined node dictionary. + /// Adds or replaces a node in the address space. + /// Removes a node from the address space. + public PredefinedNodesAddressSpace( + ISystemContext context, + NodeIdDictionary predefinedNodes, + Func addAsync, + Func> removeAsync) + { + m_context = context ?? throw new ArgumentNullException(nameof(context)); + m_predefinedNodes = predefinedNodes ?? throw new ArgumentNullException(nameof(predefinedNodes)); + m_addAsync = addAsync ?? throw new ArgumentNullException(nameof(addAsync)); + m_removeAsync = removeAsync ?? throw new ArgumentNullException(nameof(removeAsync)); + } + + /// + public ISystemContext Context => m_context; + + /// + public IEnumerable Nodes + { + get + { + var nodes = new List(); + foreach (NodeState node in m_predefinedNodes.Values) + { + // Skip instance children whose parent is already a + // top-level predefined node; they travel with the parent. + if (node is BaseInstanceState instance && + instance.Parent != null && + !instance.Parent.NodeId.IsNull && + m_predefinedNodes.ContainsKey(instance.Parent.NodeId)) + { + continue; + } + + nodes.Add(node); + } + + return nodes; + } + } + + /// + public event Action? NodeAdded; + + /// + public event Action? NodeRemoved; + + /// + public bool TryGetNode(NodeId nodeId, [NotNullWhen(true)] out NodeState? node) + { + return m_predefinedNodes.TryGetValue(nodeId, out node); + } + + /// + public async ValueTask AddOrUpdateNodeAsync(NodeState node, CancellationToken cancellationToken = default) + { + if (node == null) + { + throw new ArgumentNullException(nameof(node)); + } + + await m_addAsync(node, cancellationToken).ConfigureAwait(false); + NodeAdded?.Invoke(node); + } + + /// + public async ValueTask RemoveNodeAsync(NodeId nodeId, CancellationToken cancellationToken = default) + { + bool removed = await m_removeAsync(nodeId, cancellationToken).ConfigureAwait(false); + if (removed) + { + NodeRemoved?.Invoke(nodeId); + } + + return removed; + } + + private readonly ISystemContext m_context; + private readonly NodeIdDictionary m_predefinedNodes; + private readonly Func m_addAsync; + private readonly Func> m_removeAsync; + } +} diff --git a/Libraries/Opc.Ua.Server/Server/IGetEndpointsDirector.cs b/Libraries/Opc.Ua.Server/Server/IGetEndpointsDirector.cs new file mode 100644 index 0000000000..86a0224297 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Server/IGetEndpointsDirector.cs @@ -0,0 +1,59 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server +{ + /// + /// Optional seam consulted by GetEndpoints to direct a Client to a different member of a + /// RedundantServerSet (an extension beyond OPC 10000-4 §6.6). It is opt-in and gated: normal discovery is + /// unaffected, and the standard client-driven RedundantServerArray / ServiceLevel selection remains + /// the authoritative Failover mechanism. + /// + public interface IGetEndpointsDirector + { + /// + /// Decides whether a GetEndpoints request should be answered with a peer Server's endpoints instead of + /// the local Server's. + /// + /// The URL the Client used to reach the discovery endpoint. + /// The endpoints the local Server would otherwise return. + /// Cancellation token. + /// + /// Redirect = true and the peer's Endpoints when the request should be directed to a peer; + /// otherwise Redirect = false and the local Server serves its own endpoints. + /// + ValueTask<(bool Redirect, ArrayOf Endpoints)> TryGetDirectedEndpointsAsync( + string? endpointUrl, + ArrayOf localEndpoints, + CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Server/Server/IRedundantServerSetProvider.cs b/Libraries/Opc.Ua.Server/Server/IRedundantServerSetProvider.cs new file mode 100644 index 0000000000..c310e98455 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Server/IRedundantServerSetProvider.cs @@ -0,0 +1,44 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Server +{ + /// + /// Provides peer descriptions for a non-transparent redundant server set. + /// + public interface IRedundantServerSetProvider + { + /// + /// Gets peer entries returned by + /// FindServers for redundant server discovery. + /// + /// The peer server descriptions. + ArrayOf GetRedundantServerSet(); + } +} diff --git a/Libraries/Opc.Ua.Server/Server/StandardServer.cs b/Libraries/Opc.Ua.Server/Server/StandardServer.cs index 992fe9bee1..550fd64058 100644 --- a/Libraries/Opc.Ua.Server/Server/StandardServer.cs +++ b/Libraries/Opc.Ua.Server/Server/StandardServer.cs @@ -71,6 +71,19 @@ public StandardServer(ITelemetryContext telemetry, TimeProvider? timeProvider) /// protected TimeProvider TimeProvider { get; } + /// + /// Gets or sets the optional redundant server-set provider consulted by + /// FindServers. + /// + public IRedundantServerSetProvider? RedundantServerSetProvider { get; set; } + + /// + /// Gets or sets the optional load-direction seam consulted by + /// GetEndpoints to direct a Client to a peer Server (extension + /// beyond OPC 10000-4 §6.6; opt-in and gated). + /// + public IGetEndpointsDirector? GetEndpointsDirector { get; set; } + /// protected override void Dispose(bool disposing) { @@ -173,6 +186,8 @@ public override async ValueTask FindServersAsync( // add to list of servers to return. servers.Add(application); } + + AddRedundantServerSetDescriptions(serverUris, uniqueServers, servers); } finally { @@ -186,6 +201,32 @@ public override async ValueTask FindServersAsync( }; } + private void AddRedundantServerSetDescriptions( + ArrayOf serverUris, + Dictionary uniqueServers, + List servers) + { + IRedundantServerSetProvider? provider = RedundantServerSetProvider; + if (provider == null) + { + return; + } + + foreach (ApplicationDescription peer in provider.GetRedundantServerSet()) + { + string? applicationUri = peer.ApplicationUri; + if (string.IsNullOrEmpty(applicationUri) || + uniqueServers.ContainsKey(applicationUri) || + (serverUris.Count > 0 && !serverUris.Contains(uri => uri == applicationUri))) + { + continue; + } + + uniqueServers.Add(applicationUri, peer); + servers.Add(peer); + } + } + /// public override async ValueTask GetEndpointsAsync( SecureChannelContext secureChannelContext, @@ -213,6 +254,21 @@ public override async ValueTask GetEndpointsAsync( m_semaphoreSlim.Release(); } + // consult the optional load-direction seam; when it directs the + // Client to a peer it replaces the local endpoints (extension + // beyond OPC 10000-4 §6.6; opt-in and gated). + IGetEndpointsDirector? director = GetEndpointsDirector; + if (director != null) + { + (bool redirect, ArrayOf directed) = await director + .TryGetDirectedEndpointsAsync(endpointUrl, endpoints, requestLifetime.CancellationToken) + .ConfigureAwait(false); + if (redirect) + { + endpoints = directed; + } + } + return new GetEndpointsResponse { ResponseHeader = CreateResponse(requestHeader, StatusCodes.Good), @@ -3962,6 +4018,15 @@ protected virtual EventManager CreateEventManager( (uint)configuration.ServerConfiguration.MaxDurableEventQueueSize); } + /// + /// An optional factory used to build the server's session manager. When + /// null (the default), the built-in + /// is used. Set this before the server starts (the hosting layer wires + /// it from dependency injection) to plug in a custom session manager, + /// e.g. a distributed one for high availability. + /// + public ISessionManagerFactory? SessionManagerFactory { get; set; } + /// /// Creates the session manager for the server. /// @@ -3972,9 +4037,23 @@ protected virtual ISessionManager CreateSessionManager( IServerInternal server, ApplicationConfiguration configuration) { + if (SessionManagerFactory != null) + { + return SessionManagerFactory.Create( + server, configuration, TimeProvider, GetInstanceCertificateForPolicy); + } return new SessionManager(server, configuration, TimeProvider); } + /// + /// Resolves the server's instance certificate for a security policy URI, + /// for use by a custom session manager factory. + /// + private Certificate? GetInstanceCertificateForPolicy(string securityPolicyUri) + { + return CertificateManager?.GetInstanceCertificate(securityPolicyUri)?.Certificate; + } + /// /// Creates the session manager for the server. /// diff --git a/Libraries/Opc.Ua.Server/Session/ISessionManagerFactory.cs b/Libraries/Opc.Ua.Server/Session/ISessionManagerFactory.cs new file mode 100644 index 0000000000..57089f2e90 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Session/ISessionManagerFactory.cs @@ -0,0 +1,67 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using Opc.Ua.Security.Certificates; + +namespace Opc.Ua.Server +{ + /// + /// Factory for the server's . + /// + /// + /// Register an implementation (e.g. via + /// StandardServer.SessionManagerFactory) so the server builds a + /// custom session manager — for example a distributed one that mirrors + /// session state across replicas — instead of the default + /// . This keeps the customization additive: the + /// default path is unchanged when no factory is supplied. + /// + public interface ISessionManagerFactory + { + /// + /// Creates the session manager for the server. + /// + /// The hosting server. + /// The application configuration. + /// The server time provider. + /// + /// Resolves the server's instance certificate for a given security + /// policy URI (used by distributed managers to reconstruct a mirrored + /// session). May return null when no certificate is configured + /// for the policy. + /// + /// The session manager to use. + ISessionManager Create( + IServerInternal server, + ApplicationConfiguration configuration, + TimeProvider timeProvider, + Func serverCertificateProvider); + } +} diff --git a/Libraries/Opc.Ua.Server/Session/Persistence/ContinuationPointEnvelope.cs b/Libraries/Opc.Ua.Server/Session/Persistence/ContinuationPointEnvelope.cs new file mode 100644 index 0000000000..7172b6add9 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Session/Persistence/ContinuationPointEnvelope.cs @@ -0,0 +1,107 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +namespace Opc.Ua.Server +{ + /// + /// Serializable envelope for a mirrored continuation point. + /// + /// + /// The envelope mirrors the continuation-point id, owning session id, and re-issuable request parameters. + /// Generic server mirroring does not reconstruct node-manager-owned opaque + /// on a backup replica. After failover, a mirrored-but-unreconstructable continuation point is recognized and + /// fails gracefully with so the client re-issues Browse or + /// HistoryRead, as permitted by OPC UA Part 4 §6.6.2.2. Node managers that can serialize their own continuation + /// data may use this seam to add full-fidelity restoration. + /// + public sealed class ContinuationPointEnvelope + { + /// + /// The continuation point id. + /// + public Guid Id { get; init; } + + /// + /// The session id that owned the continuation point on the active replica. + /// + public NodeId OwnerSessionId { get; init; } + + /// + /// The continuation point family. + /// + public ContinuationPointKind Kind { get; init; } + + /// + /// The original browse target, when available. + /// + public NodeId BrowseNodeId { get; init; } + + /// + /// The browse view, when available. + /// + public ViewDescription? View { get; init; } + + /// + /// The requested maximum references per Browse result. + /// + public uint MaxResultsToReturn { get; init; } + + /// + /// The requested browse direction. + /// + public BrowseDirection BrowseDirection { get; init; } + + /// + /// The requested reference type filter. + /// + public NodeId ReferenceTypeId { get; init; } + + /// + /// Whether reference subtypes were included. + /// + public bool IncludeSubtypes { get; init; } + + /// + /// The requested node-class mask. + /// + public uint NodeClassMask { get; init; } + + /// + /// The requested browse result mask. + /// + public BrowseResultMask ResultMask { get; init; } + + /// + /// The index at which the original replica paused. + /// + public int Index { get; init; } + } +} diff --git a/Libraries/Opc.Ua.Server/Session/Persistence/ContinuationPointKind.cs b/Libraries/Opc.Ua.Server/Session/Persistence/ContinuationPointKind.cs new file mode 100644 index 0000000000..b49e9a8f5b --- /dev/null +++ b/Libraries/Opc.Ua.Server/Session/Persistence/ContinuationPointKind.cs @@ -0,0 +1,47 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Server +{ + /// + /// Identifies the continuation point family stored in a continuation-point mirror. + /// + public enum ContinuationPointKind + { + /// + /// Browse or BrowseNext continuation point. + /// + Browse = 0, + + /// + /// HistoryRead continuation point. + /// + History = 1 + } +} diff --git a/Libraries/Opc.Ua.Server/Session/Persistence/IContinuationPointStore.cs b/Libraries/Opc.Ua.Server/Session/Persistence/IContinuationPointStore.cs new file mode 100644 index 0000000000..5a6b35987f --- /dev/null +++ b/Libraries/Opc.Ua.Server/Session/Persistence/IContinuationPointStore.cs @@ -0,0 +1,75 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server +{ + /// + /// Optional mirror for best-effort continuation point envelopes. + /// + /// + /// Continuation points are mirrored as a best-effort envelope. Built-in node managers' opaque + /// is not reconstructed on a backup replica, so after failover a client + /// re-issues Browse or HistoryRead when a mirrored continuation point returns + /// . This behavior is permitted by OPC UA Part 4 §6.6.2.2, + /// which requires clients to be prepared for lost continuation points. Node managers may opt in to full + /// continuation point data serialization by storing reconstructable data through this seam. + /// + public interface IContinuationPointStore + { + /// + /// Stores or updates a continuation point envelope. + /// + /// The envelope to store. + void StoreContinuationPoint(ContinuationPointEnvelope envelope); + + /// + /// Removes a continuation point envelope. + /// + /// The owning session id. + /// The continuation point kind. + /// The continuation point id. + void RemoveContinuationPoint(NodeId ownerSessionId, ContinuationPointKind kind, Guid id); + + /// + /// Loads the mirrored continuation point envelopes for a restored session. + /// + /// + /// The session id that owned the continuation points on the active replica. + /// + /// The cancellation token. + /// The mirrored envelopes. + ValueTask> LoadContinuationPointsAsync( + NodeId ownerSessionId, + CancellationToken cancellationToken = default); + } +} diff --git a/Libraries/Opc.Ua.Server/Session/Session.cs b/Libraries/Opc.Ua.Server/Session/Session.cs index 8c43748c85..3cfc803b25 100644 --- a/Libraries/Opc.Ua.Server/Session/Session.cs +++ b/Libraries/Opc.Ua.Server/Session/Session.cs @@ -160,8 +160,11 @@ public Session( m_clientIssuerCertificates = clientCertificateChain; SecureChannelId = context.ChannelContext.SecureChannelId; - MaxBrowseContinuationPoints = maxBrowseContinuationPoints; - m_maxHistoryContinuationPoints = maxHistoryContinuationPoints; + m_continuationPoints = new SessionContinuationPoints( + () => Id, + maxBrowseContinuationPoints, + maxHistoryContinuationPoints, + server.SubscriptionStore as IContinuationPointStore); EndpointDescription = context.ChannelContext.EndpointDescription!; // use anonymous the default identity. @@ -258,35 +261,7 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - List? browseCPs; - lock (m_lock) - { - browseCPs = m_browseContinuationPoints; - m_browseContinuationPoints = null; - } - - if (browseCPs != null) - { - for (int ii = 0; ii < browseCPs.Count; ii++) - { - browseCPs[ii]?.Dispose(); - } - } - - List? historyCPs; - lock (m_lock) - { - historyCPs = m_historyContinuationPoints; - m_historyContinuationPoints = null; - } - - if (historyCPs != null) - { - for (int ii = 0; ii < historyCPs.Count; ii++) - { - (historyCPs[ii].Value as IDisposable)?.Dispose(); - } - } + m_continuationPoints.Clear(); m_userTokenNonce?.Dispose(); m_userTokenNonce = null; @@ -474,7 +449,11 @@ public virtual void SetUserTokenSecurityPolicy(string securityPolicyUri) /// /// allow derived classes access /// - protected int MaxBrowseContinuationPoints { get; set; } + protected int MaxBrowseContinuationPoints + { + get => m_continuationPoints.MaxBrowse; + set => m_continuationPoints.MaxBrowse = value; + } /// /// Validates the request. @@ -774,26 +753,7 @@ await m_server.DiagnosticsNodeManager /// is null. public void SaveContinuationPoint(ContinuationPoint continuationPoint) { - if (continuationPoint == null) - { - throw new ArgumentNullException(nameof(continuationPoint)); - } - - lock (m_lock) - { - m_browseContinuationPoints ??= []; - - // remove the first continuation point if too many points. - while (m_browseContinuationPoints.Count > MaxBrowseContinuationPoints) - { - ContinuationPoint cp = m_browseContinuationPoints[0]; - m_browseContinuationPoints.RemoveAt(0); - cp?.Dispose(); - } - - // add to end of list. - m_browseContinuationPoints.Add(continuationPoint); - } + m_continuationPoints.SaveBrowse(continuationPoint); } /// @@ -804,32 +764,7 @@ public void SaveContinuationPoint(ContinuationPoint continuationPoint) /// public ContinuationPoint? RestoreContinuationPoint(ByteString continuationPoint) { - lock (m_lock) - { - if (m_browseContinuationPoints == null) - { - return null; - } - - if (continuationPoint.Length != 16) - { - return null; - } - - var id = new Guid(continuationPoint.ToArray()); - - for (int ii = 0; ii < m_browseContinuationPoints.Count; ii++) - { - if (m_browseContinuationPoints[ii].Id == id) - { - ContinuationPoint cp = m_browseContinuationPoints[ii]; - m_browseContinuationPoints.RemoveAt(ii); - return cp; - } - } - - return null; - } + return m_continuationPoints.RestoreBrowse(continuationPoint); } /// @@ -844,33 +779,7 @@ public void SaveContinuationPoint(ContinuationPoint continuationPoint) /// is null. public void SaveHistoryContinuationPoint(Guid id, object continuationPoint) { - if (continuationPoint == null) - { - throw new ArgumentNullException(nameof(continuationPoint)); - } - - lock (m_lock) - { - m_historyContinuationPoints ??= []; - - // remove existing continuation point if space needed. - while (m_historyContinuationPoints.Count >= m_maxHistoryContinuationPoints) - { - HistoryContinuationPoint oldCP = m_historyContinuationPoints[0]; - m_historyContinuationPoints.RemoveAt(0); - (oldCP.Value as IDisposable)?.Dispose(); - } - - // create the cp. - var cp = new HistoryContinuationPoint - { - Id = id, - Value = continuationPoint, - Timestamp = DateTime.UtcNow - }; - - m_historyContinuationPoints.Add(cp); - } + m_continuationPoints.SaveHistory(id, continuationPoint); } /// @@ -880,43 +789,19 @@ public void SaveHistoryContinuationPoint(Guid id, object continuationPoint) /// The save continuation point. null if not found. public object? RestoreHistoryContinuationPoint(ByteString continuationPoint) { - lock (m_lock) - { - if (m_historyContinuationPoints == null) - { - return null; - } - - if (continuationPoint.Length != 16) - { - return null; - } - - var id = new Guid(continuationPoint.ToArray()); - - for (int ii = 0; ii < m_historyContinuationPoints.Count; ii++) - { - HistoryContinuationPoint cp = m_historyContinuationPoints[ii]; - - if (cp.Id == id) - { - m_historyContinuationPoints.RemoveAt(ii); - return cp.Value; - } - } - - return null; - } + return m_continuationPoints.RestoreHistory(continuationPoint); } /// - /// Stores a continuation point used for historial reads. + /// Loads mirrored continuation point envelopes for a session restored on a backup replica. /// - private class HistoryContinuationPoint + /// The original owner session id from the active replica. + /// The cancellation token. + public ValueTask LoadMirroredContinuationPointsAsync( + NodeId ownerSessionId, + CancellationToken cancellationToken = default) { - public Guid Id; - public object? Value; - public DateTime Timestamp; + return m_continuationPoints.LoadMirroredAsync(ownerSessionId, cancellationToken); } /// @@ -1438,10 +1323,8 @@ private void UpdateDiagnosticCounters( private string? m_userTokenSecurityPolicyUri; private Nonce? m_userTokenNonce; private readonly CertificateCollection? m_clientIssuerCertificates; - private readonly int m_maxHistoryContinuationPoints; + private readonly SessionContinuationPoints m_continuationPoints; private readonly SessionSecurityDiagnosticsDataType m_securityDiagnostics; - private List? m_browseContinuationPoints; - private List? m_historyContinuationPoints; private long m_lastContactTickCount; private int m_identityStale; } diff --git a/Libraries/Opc.Ua.Server/Session/SessionContinuationPoints.cs b/Libraries/Opc.Ua.Server/Session/SessionContinuationPoints.cs new file mode 100644 index 0000000000..c90c740280 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Session/SessionContinuationPoints.cs @@ -0,0 +1,352 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server +{ + /// + /// Encapsulates a session's browse and history continuation points: their in-memory lists, the mirrored-owner + /// bookkeeping used by a redundant standby, and the optional that persists + /// them for cross-replica takeover. Keeping this here lets delegate through a small surface + /// (save/restore/load/clear) instead of managing the store, lists, and dictionaries inline. + /// + internal sealed class SessionContinuationPoints + { + /// + /// Creates the continuation-point holder for a session. + /// + /// Returns the owning session's id (read lazily so it is current). + /// The maximum number of browse continuation points retained. + /// The maximum number of history continuation points retained. + /// + /// Optional store that mirrors continuation points across a RedundantServerSet; null when the + /// server is not distributed. + /// + public SessionContinuationPoints( + Func sessionIdProvider, + int maxBrowse, + int maxHistory, + IContinuationPointStore? store) + { + m_sessionIdProvider = sessionIdProvider ?? throw new ArgumentNullException(nameof(sessionIdProvider)); + MaxBrowse = maxBrowse; + m_maxHistory = maxHistory; + m_store = store; + } + + /// + /// Gets or sets the maximum number of browse continuation points retained before the oldest is dropped. + /// + public int MaxBrowse { get; set; } + + /// + /// Saves a browse continuation point, dropping the oldest when the limit is exceeded. + /// + /// is null. + public void SaveBrowse(ContinuationPoint continuationPoint) + { + if (continuationPoint == null) + { + throw new ArgumentNullException(nameof(continuationPoint)); + } + + lock (m_lock) + { + m_browse ??= []; + + // remove the first continuation point if too many points. + while (m_browse.Count > MaxBrowse) + { + ContinuationPoint cp = m_browse[0]; + m_browse.RemoveAt(0); + m_store?.RemoveContinuationPoint(Id, ContinuationPointKind.Browse, cp.Id); + cp?.Dispose(); + } + + // add to end of list. + m_browse.Add(continuationPoint); + } + + m_store?.StoreContinuationPoint(CreateBrowseEnvelope(continuationPoint)); + } + + /// + /// Restores (and removes) a browse continuation point. The caller disposes the returned point. + /// + public ContinuationPoint? RestoreBrowse(ByteString continuationPoint) + { + lock (m_lock) + { + if (m_browse == null) + { + return null; + } + + if (continuationPoint.Length != 16) + { + return null; + } + + var id = new Guid(continuationPoint.ToArray()); + + for (int ii = 0; ii < m_browse.Count; ii++) + { + if (m_browse[ii].Id == id) + { + ContinuationPoint cp = m_browse[ii]; + m_browse.RemoveAt(ii); + m_store?.RemoveContinuationPoint(Id, ContinuationPointKind.Browse, id); + return cp; + } + } + + if (m_mirroredBrowseOwners != null && + m_mirroredBrowseOwners.TryGetValue(id, out NodeId ownerSessionId)) + { + m_mirroredBrowseOwners.Remove(id); + m_store?.RemoveContinuationPoint(ownerSessionId, ContinuationPointKind.Browse, id); + } + + return null; + } + } + + /// + /// Saves a history continuation point, dropping the oldest when the limit is reached. A point that implements + /// is disposed when it is dropped or the session is cleared. + /// + /// is null. + public void SaveHistory(Guid id, object continuationPoint) + { + if (continuationPoint == null) + { + throw new ArgumentNullException(nameof(continuationPoint)); + } + + lock (m_lock) + { + m_history ??= []; + + // remove existing continuation point if space needed. + while (m_history.Count >= m_maxHistory) + { + HistoryContinuationPoint oldCP = m_history[0]; + m_history.RemoveAt(0); + m_store?.RemoveContinuationPoint(Id, ContinuationPointKind.History, oldCP.Id); + (oldCP.Value as IDisposable)?.Dispose(); + } + + // create the cp. + var cp = new HistoryContinuationPoint + { + Id = id, + Value = continuationPoint, + Timestamp = DateTime.UtcNow + }; + + m_history.Add(cp); + } + + m_store?.StoreContinuationPoint(CreateHistoryEnvelope(id)); + } + + /// + /// Restores (and removes) a previously saved history continuation point, or null when not found. + /// + public object? RestoreHistory(ByteString continuationPoint) + { + lock (m_lock) + { + if (m_history == null) + { + return null; + } + + if (continuationPoint.Length != 16) + { + return null; + } + + var id = new Guid(continuationPoint.ToArray()); + + for (int ii = 0; ii < m_history.Count; ii++) + { + HistoryContinuationPoint cp = m_history[ii]; + + if (cp.Id == id) + { + m_history.RemoveAt(ii); + m_store?.RemoveContinuationPoint(Id, ContinuationPointKind.History, id); + return cp.Value; + } + } + + if (m_mirroredHistoryOwners != null && + m_mirroredHistoryOwners.TryGetValue(id, out NodeId ownerSessionId)) + { + m_mirroredHistoryOwners.Remove(id); + m_store?.RemoveContinuationPoint(ownerSessionId, ContinuationPointKind.History, id); + } + + return null; + } + } + + /// + /// Loads mirrored continuation-point envelopes for a session restored on a backup replica, recording the + /// original owner so the entry can be cleaned from the shared store when it is consumed. + /// + /// The original owner session id from the active replica. + /// The cancellation token. + public async ValueTask LoadMirroredAsync( + NodeId ownerSessionId, + CancellationToken cancellationToken = default) + { + if (m_store == null || ownerSessionId.IsNull) + { + return; + } + + ArrayOf envelopes = await m_store + .LoadContinuationPointsAsync(ownerSessionId, cancellationToken) + .ConfigureAwait(false); + + lock (m_lock) + { + foreach (ContinuationPointEnvelope envelope in envelopes) + { + switch (envelope.Kind) + { + case ContinuationPointKind.Browse: + m_mirroredBrowseOwners ??= []; + m_mirroredBrowseOwners[envelope.Id] = envelope.OwnerSessionId; + break; + case ContinuationPointKind.History: + m_mirroredHistoryOwners ??= []; + m_mirroredHistoryOwners[envelope.Id] = envelope.OwnerSessionId; + break; + } + } + } + } + + /// + /// Removes and disposes all continuation points (called when the session is closed or discarded). + /// + public void Clear() + { + List? browseCPs; + List? historyCPs; + lock (m_lock) + { + browseCPs = m_browse; + m_browse = null; + historyCPs = m_history; + m_history = null; + } + + if (browseCPs != null) + { + for (int ii = 0; ii < browseCPs.Count; ii++) + { + ContinuationPoint cp = browseCPs[ii]; + m_store?.RemoveContinuationPoint(Id, ContinuationPointKind.Browse, cp.Id); + cp.Dispose(); + } + } + + if (historyCPs != null) + { + for (int ii = 0; ii < historyCPs.Count; ii++) + { + m_store?.RemoveContinuationPoint(Id, ContinuationPointKind.History, historyCPs[ii].Id); + (historyCPs[ii].Value as IDisposable)?.Dispose(); + } + } + } + + private ContinuationPointEnvelope CreateBrowseEnvelope(ContinuationPoint continuationPoint) + { + return new ContinuationPointEnvelope + { + Id = continuationPoint.Id, + OwnerSessionId = Id, + Kind = ContinuationPointKind.Browse, + BrowseNodeId = NormalizeNodeId(continuationPoint.RequestedNodeId), + View = continuationPoint.View, + MaxResultsToReturn = continuationPoint.MaxResultsToReturn, + BrowseDirection = continuationPoint.BrowseDirection, + ReferenceTypeId = NormalizeNodeId(continuationPoint.ReferenceTypeId), + IncludeSubtypes = continuationPoint.IncludeSubtypes, + NodeClassMask = continuationPoint.NodeClassMask, + ResultMask = continuationPoint.ResultMask, + Index = continuationPoint.Index + }; + } + + private ContinuationPointEnvelope CreateHistoryEnvelope(Guid id) + { + return new ContinuationPointEnvelope + { + Id = id, + OwnerSessionId = Id, + Kind = ContinuationPointKind.History, + BrowseNodeId = NodeId.Null, + ReferenceTypeId = NodeId.Null + }; + } + + private static NodeId NormalizeNodeId(NodeId nodeId) + { + return nodeId.IsNull ? NodeId.Null : nodeId; + } + + private NodeId Id => m_sessionIdProvider(); + + private sealed class HistoryContinuationPoint + { + public Guid Id; + public object? Value; + public DateTime Timestamp; + } + + private readonly Func m_sessionIdProvider; + private readonly int m_maxHistory; + private readonly IContinuationPointStore? m_store; + private readonly Lock m_lock = new(); + private List? m_browse; + private List? m_history; + private Dictionary? m_mirroredBrowseOwners; + private Dictionary? m_mirroredHistoryOwners; + } +} diff --git a/Libraries/Opc.Ua.Server/Session/SessionManager.cs b/Libraries/Opc.Ua.Server/Session/SessionManager.cs index 4cf8bf7c72..949d3e2c94 100644 --- a/Libraries/Opc.Ua.Server/Session/SessionManager.cs +++ b/Libraries/Opc.Ua.Server/Session/SessionManager.cs @@ -360,7 +360,7 @@ await session.InitializeAsync(context, cancellationToken) string? clientKey = null; // fast path no lock - if (!m_sessions.TryGetValue(authenticationToken, out _)) + if (!m_sessions.TryGetValue(authenticationToken, out _) && !SupportsSessionRestore) { throw new ServiceResultException(StatusCodes.BadSessionIdInvalid); } @@ -373,7 +373,23 @@ await session.InitializeAsync(context, cancellationToken) // find session. if (!m_sessions.TryGetValue(authenticationToken, out session)) { - throw new ServiceResultException(StatusCodes.BadSessionIdInvalid); + // Not present locally. In a distributed deployment this + // is a failover reconnect: let a derived manager restore + // the mirrored session so the standard ActivateSession + // validation below (full client-signature check against + // the restored, single-use serverNonce) still runs. The + // token is a lookup key only, never an authenticator. + session = await RestoreSessionAsync( + authenticationToken, context, cancellationToken).ConfigureAwait(false); + if (session == null) + { + throw new ServiceResultException(StatusCodes.BadSessionIdInvalid); + } + if (!m_sessions.TryAdd(authenticationToken, session) && + !m_sessions.TryGetValue(authenticationToken, out session)) + { + throw new ServiceResultException(StatusCodes.BadSessionIdInvalid); + } } // get client lockout key. @@ -1016,6 +1032,44 @@ private void EnsureRoleManagerSubscription() current?.RoleConfigurationChanged += OnRoleConfigurationChanged; } + /// + /// Gets a value indicating whether this manager can restore sessions + /// that are not present in the local session table (e.g. a mirrored + /// session after a failover). When false (the default), an + /// ActivateSession for an unknown token fails fast with + /// exactly as before. + /// + protected virtual bool SupportsSessionRestore => false; + + /// + /// Restores a session that is not present in the local session table. + /// + /// + /// Called by (under the session lock) + /// when the supplied is unknown + /// locally. The default returns null so the activation is + /// rejected. A distributed manager overrides this to reconstruct a + /// mirrored session (e.g. from a shared store), after which the normal + /// activation path performs the full client-certificate signature + /// validation — the token is never an authenticator on its own. The + /// returned session must be fully initialized (its diagnostics node + /// registered, i.e. awaited). + /// + /// The unknown authentication token. + /// The operation context of the activation. + /// The cancellation token. + /// + /// The restored session to admit to the local table, or null to + /// reject the activation. + /// + protected virtual ValueTask RestoreSessionAsync( + NodeId authenticationToken, + OperationContext context, + CancellationToken cancellationToken = default) + { + return new ValueTask((ISession?)null); + } + /// /// Creates a new instance of a session. /// diff --git a/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionRetransmissionDeltaStore.cs b/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionRetransmissionDeltaStore.cs new file mode 100644 index 0000000000..c620e9a4ad --- /dev/null +++ b/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionRetransmissionDeltaStore.cs @@ -0,0 +1,54 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Server +{ + /// + /// Optional delta path for live retransmission mirrors. + /// + /// + /// Implementations avoid per-publish full retransmission queue snapshots by accepting only the notifications + /// added to and removed from the queue since the previous mirror update. + /// + public interface ISubscriptionRetransmissionDeltaStore : ISubscriptionRetransmissionStore + { + /// + /// Stores a retransmission queue delta for a subscription. + /// + /// The subscription id. + /// The next sequence number to assign. + /// The newly sent notifications available for republish. + /// Sequence numbers no longer available for republish. + void StoreRetransmissionStateDelta( + uint subscriptionId, + uint nextSequenceNumber, + ArrayOf addedMessages, + ArrayOf removedSequenceNumbers); + } +} diff --git a/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionRetransmissionStore.cs b/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionRetransmissionStore.cs new file mode 100644 index 0000000000..06a55069de --- /dev/null +++ b/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionRetransmissionStore.cs @@ -0,0 +1,72 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Server +{ + /// + /// Optional live retransmission mirror for subscription sequence numbers and sent notifications. + /// + /// + /// Implementations are used only when a server's also implements this interface. + /// Single-replica servers that do not configure a provider keep the existing in-memory retransmission behavior. + /// + public interface ISubscriptionRetransmissionStore + { + /// + /// Loads the latest mirrored retransmission state for a subscription. + /// + /// The subscription id. + /// The cancellation token. + /// The restored retransmission state, or null when no mirrored state exists. + ValueTask LoadRetransmissionStateAsync( + uint subscriptionId, + CancellationToken cancellationToken = default); + + /// + /// Stores the current retransmission snapshot for a subscription. + /// + /// The subscription id. + /// The next sequence number to assign. + /// The sent notifications still available for republish. + void StoreRetransmissionState( + uint subscriptionId, + uint nextSequenceNumber, + ArrayOf sentMessages); + + /// + /// Removes an acknowledged notification from the mirrored retransmission cache. + /// + /// The subscription id. + /// The acknowledged sequence number. + void AcknowledgeNotification(uint subscriptionId, uint sequenceNumber); + } +} diff --git a/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionStore.cs b/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionStore.cs index c50f6b0b8d..45c16a9069 100644 --- a/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionStore.cs +++ b/Libraries/Opc.Ua.Server/Subscription/Persistence/ISubscriptionStore.cs @@ -28,6 +28,8 @@ * ======================================================================*/ using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; namespace Opc.Ua.Server { @@ -37,17 +39,22 @@ namespace Opc.Ua.Server public interface ISubscriptionStore { /// - /// Restore subscriptions from storage, called on server startup + /// Restore subscriptions from storage, called on server startup. /// + /// a token to cancel the operation /// the result of the restore operation - RestoreSubscriptionResult RestoreSubscriptions(); + ValueTask RestoreSubscriptionsAsync( + CancellationToken cancellationToken = default); /// - /// Store subscriptions in storage, called on server shutdown + /// Store subscriptions in storage, called on server shutdown. /// /// the subscription templates to store + /// a token to cancel the operation /// true if storing was successful - bool StoreSubscriptions(IEnumerable subscriptions); + ValueTask StoreSubscriptionsAsync( + IEnumerable subscriptions, + CancellationToken cancellationToken = default); /// /// Restore a DataChangeMonitoredItemQueue from storage @@ -67,8 +74,11 @@ public interface ISubscriptionStore /// Signals created Subscription ids incl. MonitoredItem ids to the SubscriptionStore instance, to signal cleanup can take place /// The store shall clean all stored subscriptions, monitoredItems, and only keep the persitent queues for the monitoredItem ids provided /// key = subscription id, value = monitoredItem ids + /// a token to cancel the operation /// - void OnSubscriptionRestoreComplete(Dictionary> createdSubscriptions); + ValueTask OnSubscriptionRestoreCompleteAsync( + Dictionary> createdSubscriptions, + CancellationToken cancellationToken = default); } /// diff --git a/Libraries/Opc.Ua.Server/Subscription/Persistence/SubscriptionRetransmissionState.cs b/Libraries/Opc.Ua.Server/Subscription/Persistence/SubscriptionRetransmissionState.cs new file mode 100644 index 0000000000..c70007b554 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Subscription/Persistence/SubscriptionRetransmissionState.cs @@ -0,0 +1,47 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Server +{ + /// + /// Mirrored retransmission state for a subscription. + /// + public sealed class SubscriptionRetransmissionState + { + /// + /// The next sequence number the subscription should assign. + /// + public uint NextSequenceNumber { get; set; } + + /// + /// Sent notifications that remain available for republish. + /// + public ArrayOf SentMessages { get; set; } = []; + } +} diff --git a/Libraries/Opc.Ua.Server/Subscription/SentMessageQueue.cs b/Libraries/Opc.Ua.Server/Subscription/SentMessageQueue.cs new file mode 100644 index 0000000000..1fc3596a85 --- /dev/null +++ b/Libraries/Opc.Ua.Server/Subscription/SentMessageQueue.cs @@ -0,0 +1,439 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Opc.Ua.Server +{ + /// + /// Owns a subscription's sent-notification ring together with its sequence-number counter and the optional + /// that mirrors them for cross-replica republish. Keeping this + /// bookkeeping here lets keep Publish/Acknowledge/Republish focused on orchestration + /// while delegating the mechanical queue maintenance (sequence assignment, overflow trimming, payload recycling, + /// retransmission mirroring) to a small, intent-revealing surface. + /// + /// + /// This type is not internally synchronized. The owning serializes every access under + /// its subscription lock (or, for the restore path, before the subscription is published), exactly as the inlined + /// state did before the extraction. + /// + internal sealed class SentMessageQueue + { + /// + /// Creates an empty queue for a new subscription (next sequence number 1, no sent messages). + /// + /// Returns the owning subscription's id (read lazily so it is current). + /// The maximum number of sent messages retained for republish. + /// + /// Optional store that mirrors retransmission state across a RedundantServerSet; null when the + /// server is not distributed. + /// + /// The logger used to report queue overflow. + public SentMessageQueue( + Func subscriptionIdProvider, + uint maxMessageCount, + ISubscriptionRetransmissionStore? retransmissionStore, + ILogger logger) + : this(subscriptionIdProvider, maxMessageCount, retransmissionStore, logger, [], 1, 0) + { + } + + private SentMessageQueue( + Func subscriptionIdProvider, + uint maxMessageCount, + ISubscriptionRetransmissionStore? retransmissionStore, + ILogger logger, + List sentMessages, + uint nextSequenceNumber, + int lastSentMessage) + { + m_subscriptionIdProvider = subscriptionIdProvider + ?? throw new ArgumentNullException(nameof(subscriptionIdProvider)); + m_maxMessageCount = maxMessageCount; + m_retransmissionStore = retransmissionStore; + m_logger = logger ?? throw new ArgumentNullException(nameof(logger)); + m_sentMessages = sentMessages; + m_sequenceNumber = nextSequenceNumber; + m_lastSentMessage = lastSentMessage; + } + + /// + /// Creates a queue restored from persisted subscription state. + /// + public static SentMessageQueue CreateRestored( + Func subscriptionIdProvider, + uint maxMessageCount, + ISubscriptionRetransmissionStore? retransmissionStore, + ILogger logger, + List sentMessages, + uint nextSequenceNumber, + int lastSentMessage) + { + return new SentMessageQueue( + subscriptionIdProvider, + maxMessageCount, + retransmissionStore, + logger, + sentMessages, + nextSequenceNumber, + lastSentMessage); + } + + /// + /// Gets the next sequence number that will be assigned (without consuming it). + /// + public uint NextSequenceNumber => m_sequenceNumber; + + /// + /// Gets the maximum number of sent messages retained for republish. + /// + public uint MaxMessageCount => m_maxMessageCount; + + /// + /// Gets the index of the next queued message that has not yet been returned to a publish request. + /// + public int LastSentMessage => m_lastSentMessage; + + /// + /// Gets the number of sent messages currently retained. + /// + public int SentCount => m_sentMessages.Count; + + /// + /// Gets the underlying sent-message list for persistence (read while the subscription is quiesced). + /// + public List SentMessages => m_sentMessages; + + /// + /// Consumes and returns the next sequence number, advancing the counter. + /// + public uint AssignSequenceNumber() + { + uint sequenceNumber = m_sequenceNumber; + Utils.IncrementIdentifier(ref m_sequenceNumber); + return sequenceNumber; + } + + /// + /// Returns the next already-queued (but not yet published) message, or null when none is queued. + /// + /// Receives the sequence numbers still available for republish. + /// Whether the subscription still has monitored-item notifications pending. + /// Set to true when more messages remain to be published. + public NotificationMessage? TryDequeueQueued( + List availableSequenceNumbers, + bool hasItemsToPublish, + out bool moreNotifications) + { + moreNotifications = false; + + if (m_lastSentMessage < m_sentMessages.Count) + { + // return the available sequence numbers. + for (int ii = 0; ii <= m_lastSentMessage && ii < m_sentMessages.Count; ii++) + { + availableSequenceNumbers.Add(m_sentMessages[ii].SequenceNumber); + } + + moreNotifications = (m_lastSentMessage < m_sentMessages.Count - 1) || hasItemsToPublish; + + return m_sentMessages[m_lastSentMessage++]; + } + + return null; + } + + /// + /// Adds the sequence numbers still available for republish to the supplied list (used for keep-alive replies). + /// + public void FillAvailableSequenceNumbers(List availableSequenceNumbers) + { + for (int ii = 0; ii <= m_lastSentMessage && ii < m_sentMessages.Count; ii++) + { + availableSequenceNumbers.Add(m_sentMessages[ii].SequenceNumber); + } + } + + /// + /// Appends freshly constructed messages to the queue, dropping/trimming as needed to respect the queue limit, + /// mirrors the retransmission state, and returns the next message to publish. + /// + /// The messages to enqueue (may be trimmed in place on overflow). + /// Receives the sequence numbers still available for republish. + /// Set to true when more messages remain to be published. + /// + /// The number of messages that displaced older unacknowledged ones (for diagnostics); 0 when the queue + /// was not full. + /// + public NotificationMessage Enqueue( + List messages, + List availableSequenceNumbers, + out bool moreNotifications, + out uint newlyUnacknowledgedCount) + { + newlyUnacknowledgedCount = 0; + + // have to drop unsent messages if out of queue space. + int overflowCount = messages.Count - (int)m_maxMessageCount; + if (overflowCount > 0) + { + m_logger.LogWarning( + "WARNING: QUEUE OVERFLOW. Dropping {Count} Messages. Increase MaxMessageQueueSize. SubId={SubscriptionId}, MaxMessageQueueSize={MaxMessageCount}", + overflowCount, + Id, + m_maxMessageCount); + for (int ii = 0; ii < overflowCount; ii++) + { + ReuseNotificationPayloads(messages[ii]); + } + messages.RemoveRange(0, overflowCount); + } + + ArrayOf removedSequenceNumbers = m_retransmissionStore == null ? default : []; + + // remove old messages if queue is full. + if (m_sentMessages.Count > m_maxMessageCount - messages.Count) + { + newlyUnacknowledgedCount = (uint)messages.Count; + + if (m_maxMessageCount <= messages.Count) + { + if (m_retransmissionStore != null) + { + removedSequenceNumbers = GetSequenceNumbers(m_sentMessages, m_sentMessages.Count); + } + for (int ii = 0; ii < m_sentMessages.Count; ii++) + { + ReuseNotificationPayloads(m_sentMessages[ii]); + } + m_sentMessages.Clear(); + } + else + { + if (m_retransmissionStore != null) + { + removedSequenceNumbers = GetSequenceNumbers(m_sentMessages, messages.Count); + } + for (int ii = 0; ii < messages.Count; ii++) + { + ReuseNotificationPayloads(m_sentMessages[ii]); + } + m_sentMessages.RemoveRange(0, messages.Count); + } + } + + // save new message + m_lastSentMessage = m_sentMessages.Count; + m_sentMessages.AddRange(messages); + StoreRetransmissionState(messages, removedSequenceNumbers); + + // check if there are more notifications to send. + moreNotifications = messages.Count > 1; + + // return the available sequence numbers. + for (int ii = 0; ii <= m_lastSentMessage && ii < m_sentMessages.Count; ii++) + { + availableSequenceNumbers.Add(m_sentMessages[ii].SequenceNumber); + } + + return m_sentMessages[m_lastSentMessage++]; + } + + /// + /// Removes an acknowledged message from the queue. + /// + /// true if the message was found and acknowledged; otherwise false. + public bool TryAcknowledge(uint sequenceNumber) + { + // find message in queue. + for (int ii = 0; ii < m_sentMessages.Count; ii++) + { + if (m_sentMessages[ii].SequenceNumber == sequenceNumber) + { + if (m_lastSentMessage > ii) + { + m_lastSentMessage--; + } + + NotificationMessage removed = m_sentMessages[ii]; + m_sentMessages.RemoveAt(ii); + ReuseNotificationPayloads(removed); + m_retransmissionStore?.AcknowledgeNotification(Id, sequenceNumber); + return true; + } + } + + return false; + } + + /// + /// Finds a previously sent message for republish, or null when it is no longer available. + /// + public NotificationMessage? FindForRepublish(uint retransmitSequenceNumber) + { + foreach (NotificationMessage sentMessage in m_sentMessages) + { + if (sentMessage.SequenceNumber == retransmitSequenceNumber) + { + return sentMessage; + } + } + + return null; + } + + /// + /// Returns the available sequence numbers for retransmission (for example used in Transfer Subscription). + /// + public ArrayOf AvailableSequenceNumbersForRetransmission() + { + var availableSequenceNumbers = new List(); + // Assumption we do not check lastSentMessage < sentMessages.Count because + // in case of subscription transfer original client might have crashed by handling message, + // therefor new client should have to chance to process all available messages + for (int ii = 0; ii < m_sentMessages.Count; ii++) + { + availableSequenceNumbers.Add(m_sentMessages[ii].SequenceNumber); + } + return availableSequenceNumbers; + } + + /// + /// Restores the retransmission state from the mirror, when one is configured and holds state. + /// + public async ValueTask LoadRetransmissionStateAsync(CancellationToken cancellationToken) + { + if (m_retransmissionStore == null) + { + return; + } + + SubscriptionRetransmissionState? state = await m_retransmissionStore + .LoadRetransmissionStateAsync(Id, cancellationToken) + .ConfigureAwait(false); + if (state == null) + { + return; + } + + m_sentMessages.Clear(); + m_sentMessages.AddRange(state.SentMessages); + m_sequenceNumber = state.NextSequenceNumber; + m_lastSentMessage = m_sentMessages.Count; + } + + /// + /// Recycles and clears all sent messages (called when the subscription is disposed). + /// + public void Clear() + { + for (int ii = 0; ii < m_sentMessages.Count; ii++) + { + ReuseNotificationPayloads(m_sentMessages[ii]); + } + m_sentMessages.Clear(); + } + + private void StoreRetransmissionState( + IList addedMessages, + ArrayOf removedSequenceNumbers) + { + if (m_retransmissionStore == null) + { + return; + } + + if (m_retransmissionStore is ISubscriptionRetransmissionDeltaStore deltaStore) + { + deltaStore.StoreRetransmissionStateDelta( + Id, + m_sequenceNumber, + new ArrayOf(addedMessages.ToArray()), + removedSequenceNumbers); + return; + } + + m_retransmissionStore.StoreRetransmissionState(Id, m_sequenceNumber, [.. m_sentMessages]); + } + + private static ArrayOf GetSequenceNumbers(List messages, int count) + { + var sequenceNumbers = new uint[count]; + for (int ii = 0; ii < count; ii++) + { + sequenceNumbers[ii] = messages[ii].SequenceNumber; + } + + return new ArrayOf(sequenceNumbers); + } + + private static void ReuseNotificationPayloads(NotificationMessage message) + { + ReadOnlySpan data = message.NotificationData.Span; + for (int i = 0; i < data.Length; i++) + { + if (data[i].TryGetValue(out DataChangeNotification? dcn)) + { + ReadOnlySpan items = + dcn.MonitoredItems.Span; + for (int j = 0; j < items.Length; j++) + { + (items[j] as IPooledEncodeable)?.Reuse(); + } + (dcn as IPooledEncodeable)?.Reuse(); + } + else if (data[i].TryGetValue(out EventNotificationList? enl)) + { + ReadOnlySpan events = enl.Events.Span; + for (int j = 0; j < events.Length; j++) + { + (events[j] as IPooledEncodeable)?.Reuse(); + } + (enl as IPooledEncodeable)?.Reuse(); + } + } + (message as IPooledEncodeable)?.Reuse(); + } + + private uint Id => m_subscriptionIdProvider(); + + private readonly Func m_subscriptionIdProvider; + private readonly uint m_maxMessageCount; + private readonly ISubscriptionRetransmissionStore? m_retransmissionStore; + private readonly ILogger m_logger; + private readonly List m_sentMessages; + private uint m_sequenceNumber; + private int m_lastSentMessage; + } +} diff --git a/Libraries/Opc.Ua.Server/Subscription/Subscription.cs b/Libraries/Opc.Ua.Server/Subscription/Subscription.cs index bfeae059d5..5f8a835807 100644 --- a/Libraries/Opc.Ua.Server/Subscription/Subscription.cs +++ b/Libraries/Opc.Ua.Server/Subscription/Subscription.cs @@ -109,8 +109,11 @@ public Subscription( m_keepAliveCounter = maxKeepAliveCount; m_lifetimeCounter = 0; m_waitingForPublish = false; - m_maxMessageCount = maxMessageCount; - m_sentMessages = []; + m_messageQueue = new SentMessageQueue( + () => Id, + maxMessageCount, + m_server.SubscriptionStore as ISubscriptionRetransmissionStore, + m_logger); m_supportsDurable = m_server.MonitoredItemQueueFactory.SupportsDurableQueues; IsDurable = false; @@ -119,10 +122,6 @@ public Subscription( m_itemsToPublish = new LinkedList(); m_itemsToTrigger = []; - // m_itemsReadyToPublish = new Queue(); - // m_itemsNotificationsAvailable = new LinkedList(); - m_sequenceNumber = 1; - // initialize diagnostics. Diagnostics = new SubscriptionDiagnosticsDataType { @@ -154,7 +153,7 @@ public Subscription( MonitoredItemCount = 0, DisabledMonitoredItemCount = 0, MonitoringQueueOverflowCount = 0, - NextSequenceNumber = m_sequenceNumber + NextSequenceNumber = m_messageQueue.NextSequenceNumber }; ServerSystemContext systemContext = m_server.DefaultSystemContext.Copy(session); @@ -189,6 +188,7 @@ public static async ValueTask RestoreAsync( CancellationToken cancellationToken = default) { var subscription = new Subscription(server, storedSubscription, timeProvider); + await subscription.LoadRetransmissionStateAsync(cancellationToken).ConfigureAwait(false); await subscription.RestoreMonitoredItemsAsync(storedSubscription.MonitoredItems, cancellationToken) .ConfigureAwait(false); @@ -237,8 +237,14 @@ protected Subscription( (long)storedSubscription.PublishingInterval; m_keepAliveCounter = 0; m_waitingForPublish = false; - m_maxMessageCount = storedSubscription.MaxMessageCount; - m_sentMessages = storedSubscription.SentMessages; + m_messageQueue = SentMessageQueue.CreateRestored( + () => Id, + storedSubscription.MaxMessageCount, + m_server.SubscriptionStore as ISubscriptionRetransmissionStore, + m_logger, + storedSubscription.SentMessages, + storedSubscription.SequenceNumber, + storedSubscription.LastSentMessage); m_supportsDurable = m_server.MonitoredItemQueueFactory.SupportsDurableQueues; IsDurable = storedSubscription.IsDurable; // UserIdentityToken is null for anonymous sessions; preserve the saved-owner @@ -246,8 +252,6 @@ protected Subscription( m_savedOwnerIdentity = storedSubscription.UserIdentityToken != null ? new UserIdentity(storedSubscription.UserIdentityToken) : null; - m_sequenceNumber = storedSubscription.SequenceNumber; - m_lastSentMessage = storedSubscription.LastSentMessage; m_monitoredItems = []; m_itemsToCheck = new LinkedList(); @@ -284,7 +288,7 @@ protected Subscription( MonitoredItemCount = 0, DisabledMonitoredItemCount = 0, MonitoringQueueOverflowCount = 0, - NextSequenceNumber = m_sequenceNumber + NextSequenceNumber = m_messageQueue.NextSequenceNumber }; ServerSystemContext systemContext = m_server.DefaultSystemContext.Copy(); @@ -321,11 +325,7 @@ protected virtual void Dispose(bool disposing) } m_monitoredItems.Clear(); - for (int ii = 0; ii < m_sentMessages.Count; ii++) - { - ReuseNotificationPayloads(m_sentMessages[ii]); - } - m_sentMessages.Clear(); + m_messageQueue.Clear(); m_itemsToCheck.Clear(); m_itemsToPublish.Clear(); } @@ -775,20 +775,9 @@ public void QueueOverflowHandler() ResetLifetimeCount(); // find message in queue. - for (int ii = 0; ii < m_sentMessages.Count; ii++) + if (m_messageQueue.TryAcknowledge(sequenceNumber)) { - if (m_sentMessages[ii].SequenceNumber == sequenceNumber) - { - if (m_lastSentMessage > ii) - { - m_lastSentMessage--; - } - - NotificationMessage removed = m_sentMessages[ii]; - m_sentMessages.RemoveAt(ii); - ReuseNotificationPayloads(removed); - return null; - } + return null; } if (sequenceNumber == 0) @@ -876,14 +865,12 @@ public NotificationMessage PublishTimeout() m_expired = true; message = (NotificationMessage)NotificationMessageActivator.Instance.CreateInstance(); - message.SequenceNumber = m_sequenceNumber; + message.SequenceNumber = m_messageQueue.AssignSequenceNumber(); message.PublishTime = DateTimeUtc.Now; - Utils.IncrementIdentifier(ref m_sequenceNumber); - lock (DiagnosticsWriteLock) { - Diagnostics.NextSequenceNumber = m_sequenceNumber; + Diagnostics.NextSequenceNumber = m_messageQueue.NextSequenceNumber; } var notification = (StatusChangeNotification)StatusChangeNotificationActivator.Instance.CreateInstance(); @@ -905,14 +892,12 @@ public NotificationMessage SubscriptionTransferred() lock (m_lock) { message = (NotificationMessage)NotificationMessageActivator.Instance.CreateInstance(); - message.SequenceNumber = m_sequenceNumber; + message.SequenceNumber = m_messageQueue.AssignSequenceNumber(); message.PublishTime = DateTimeUtc.Now; - Utils.IncrementIdentifier(ref m_sequenceNumber); - lock (DiagnosticsWriteLock) { - Diagnostics.NextSequenceNumber = m_sequenceNumber; + Diagnostics.NextSequenceNumber = m_messageQueue.NextSequenceNumber; } var notification = (StatusChangeNotification)StatusChangeNotificationActivator.Instance.CreateInstance(); @@ -944,20 +929,17 @@ public NotificationMessage SubscriptionTransferred() moreNotifications = false; - if (m_lastSentMessage < m_sentMessages.Count) + NotificationMessage? queuedMessage = m_messageQueue.TryDequeueQueued( + availableSequenceNumberList, + m_itemsToPublish.Count > 0, + out moreNotifications); + if (queuedMessage != null) { - // return the available sequence numbers. - for (int ii = 0; ii <= m_lastSentMessage && ii < m_sentMessages.Count; ii++) - { - availableSequenceNumberList.Add(m_sentMessages[ii].SequenceNumber); - } - - moreNotifications = m_waitingForPublish = (m_lastSentMessage < m_sentMessages.Count - 1) || - m_itemsToPublish.Count > 0; + m_waitingForPublish = moreNotifications; // TraceState(LogLevel.Trace, TraceStateId.Items, "PUBLISH QUEUED MESSAGE"); availableSequenceNumbers = availableSequenceNumberList; - return m_sentMessages[m_lastSentMessage++]; + return queuedMessage; } var messages = new List(); @@ -1040,8 +1022,8 @@ public NotificationMessage SubscriptionTransferred() } //stop fetching messages from MIs when message queue is full to avoid discards - // use m_maxMessageCount - 2 to put remaining values into the last allowed message (each MI is allowed to publish 3 up to messages at once) - if (messages.Count >= m_maxMessageCount - 2) + // use MaxMessageCount - 2 to put remaining values into the last allowed message (each MI is allowed to publish 3 up to messages at once) + if (messages.Count >= m_messageQueue.MaxMessageCount - 2) { break; } @@ -1104,78 +1086,41 @@ public NotificationMessage SubscriptionTransferred() { // create a keep alive message. var message = (NotificationMessage)NotificationMessageActivator.Instance.CreateInstance(); - message.SequenceNumber = m_sequenceNumber; + message.SequenceNumber = m_messageQueue.NextSequenceNumber; message.PublishTime = DateTimeUtc.Now; // return the available sequence numbers. - for (int ii = 0; ii <= m_lastSentMessage && ii < m_sentMessages.Count; ii++) - { - availableSequenceNumberList.Add(m_sentMessages[ii].SequenceNumber); - } + m_messageQueue.FillAvailableSequenceNumbers(availableSequenceNumberList); // TraceState(LogLevel.Trace, TraceStateId.Items, "PUBLISH KEEPALIVE"); availableSequenceNumbers = availableSequenceNumberList; return message; } - // have to drop unsent messages if out of queue space. - int overflowCount = messages.Count - (int)m_maxMessageCount; - if (overflowCount > 0) - { - m_logger.LogWarning( - "WARNING: QUEUE OVERFLOW. Dropping {Count} Messages. Increase MaxMessageQueueSize. SubId={SubscriptionId}, MaxMessageQueueSize={MaxMessageCount}", - overflowCount, - Id, - m_maxMessageCount); - for (int ii = 0; ii < overflowCount; ii++) - { - ReuseNotificationPayloads(messages[ii]); - } - messages.RemoveRange(0, overflowCount); - } + NotificationMessage newMessage = m_messageQueue.Enqueue( + messages, + availableSequenceNumberList, + out moreNotifications, + out uint newlyUnacknowledgedCount); - // remove old messages if queue is full. - if (m_sentMessages.Count > m_maxMessageCount - messages.Count) + if (newlyUnacknowledgedCount > 0) { lock (DiagnosticsWriteLock) { - Diagnostics.UnacknowledgedMessageCount += (uint)messages.Count; - } - - if (m_maxMessageCount <= messages.Count) - { - for (int ii = 0; ii < m_sentMessages.Count; ii++) - { - ReuseNotificationPayloads(m_sentMessages[ii]); - } - m_sentMessages.Clear(); - } - else - { - for (int ii = 0; ii < messages.Count; ii++) - { - ReuseNotificationPayloads(m_sentMessages[ii]); - } - m_sentMessages.RemoveRange(0, messages.Count); + Diagnostics.UnacknowledgedMessageCount += newlyUnacknowledgedCount; } } - // save new message - m_lastSentMessage = m_sentMessages.Count; - m_sentMessages.AddRange(messages); - - // check if there are more notifications to send. - moreNotifications = m_waitingForPublish = messages.Count > 1; - - // return the available sequence numbers. - for (int ii = 0; ii <= m_lastSentMessage && ii < m_sentMessages.Count; ii++) - { - availableSequenceNumberList.Add(m_sentMessages[ii].SequenceNumber); - } + m_waitingForPublish = moreNotifications; // TraceState(LogLevel.Trace, TraceStateId.Items, "PUBLISH NEW MESSAGE"); availableSequenceNumbers = availableSequenceNumberList; - return m_sentMessages[m_lastSentMessage++]; + return newMessage; + } + + private ValueTask LoadRetransmissionStateAsync(CancellationToken cancellationToken) + { + return m_messageQueue.LoadRetransmissionStateAsync(cancellationToken); } /// @@ -1184,15 +1129,7 @@ public NotificationMessage SubscriptionTransferred() /// public ArrayOf AvailableSequenceNumbersForRetransmission() { - var availableSequenceNumbers = new List(); - // Assumption we do not check lastSentMessage < sentMessages.Count because - // in case of subscription transfer original client might have crashed by handling message, - // therefor new client should have to chance to process all available messages - for (int ii = 0; ii < m_sentMessages.Count; ii++) - { - availableSequenceNumbers.Add(m_sentMessages[ii].SequenceNumber); - } - return availableSequenceNumbers; + return m_messageQueue.AvailableSequenceNumbersForRetransmission(); } /// @@ -1207,14 +1144,12 @@ private NotificationMessage ConstructMessage( notificationCount = 0; var message = (NotificationMessage)NotificationMessageActivator.Instance.CreateInstance(); - message.SequenceNumber = m_sequenceNumber; + message.SequenceNumber = m_messageQueue.AssignSequenceNumber(); message.PublishTime = DateTimeUtc.Now; - Utils.IncrementIdentifier(ref m_sequenceNumber); - lock (DiagnosticsWriteLock) { - Diagnostics.NextSequenceNumber = m_sequenceNumber; + Diagnostics.NextSequenceNumber = m_messageQueue.NextSequenceNumber; } // add events. @@ -1300,17 +1235,15 @@ public NotificationMessage Republish( } // find message. - foreach (NotificationMessage sentMessage in m_sentMessages) + NotificationMessage? sentMessage = m_messageQueue.FindForRepublish(retransmitSequenceNumber); + if (sentMessage != null) { - if (sentMessage.SequenceNumber == retransmitSequenceNumber) + lock (DiagnosticsWriteLock) { - lock (DiagnosticsWriteLock) - { - Diagnostics.RepublishMessageCount++; - } - - return sentMessage; + Diagnostics.RepublishMessageCount++; } + + return sentMessage; } // message not available. @@ -2613,14 +2546,14 @@ public IStoredSubscription ToStorableSubscription() return new StoredSubscription { - SentMessages = m_sentMessages, + SentMessages = m_messageQueue.SentMessages, Id = Id, - SequenceNumber = m_sequenceNumber, - LastSentMessage = m_lastSentMessage, + SequenceNumber = m_messageQueue.NextSequenceNumber, + LastSentMessage = m_messageQueue.LastSentMessage, LifetimeCounter = m_lifetimeCounter, MaxKeepaliveCount = m_maxKeepAliveCount, MaxLifetimeCount = m_maxLifetimeCount, - MaxMessageCount = m_maxMessageCount, + MaxMessageCount = m_messageQueue.MaxMessageCount, MaxNotificationsPerPublish = m_maxNotificationsPerPublish, Priority = Priority, PublishingInterval = PublishingInterval, @@ -2737,11 +2670,11 @@ private void TraceState(LogLevel logLevel, TraceStateId id, string context) // save counters Monitor.Enter(m_lock); - long sequenceNumber = m_sequenceNumber; + long sequenceNumber = m_messageQueue.NextSequenceNumber; int itemsToCheck = m_itemsToCheck.Count; int monitoredItems = m_monitoredItems.Count; int itemsToPublish = m_itemsToPublish.Count; - int sentMessages = m_sentMessages.Count; + int sentMessages = m_messageQueue.SentCount; bool publishingEnabled = m_publishingEnabled; bool waitingForPublish = m_waitingForPublish; @@ -2818,10 +2751,7 @@ private void TraceState(LogLevel logLevel, TraceStateId id, string context) private uint m_keepAliveCounter; private uint m_lifetimeCounter; private bool m_waitingForPublish; - private readonly List m_sentMessages; - private int m_lastSentMessage; - private uint m_sequenceNumber; - private readonly uint m_maxMessageCount; + private readonly SentMessageQueue m_messageQueue; private readonly Dictionary> m_monitoredItems; private readonly LinkedList m_itemsToCheck; private readonly LinkedList m_itemsToPublish; @@ -2831,39 +2761,5 @@ private void TraceState(LogLevel logLevel, TraceStateId id, string context) private readonly Dictionary> m_itemsToTrigger; private readonly bool m_supportsDurable; private readonly ILogger m_logger; - - /// - /// Walk a that is being - /// discarded (acknowledged or evicted from the retransmission - /// queue) and return every pooled notification payload to its - /// activator's pool via . - /// - private static void ReuseNotificationPayloads(NotificationMessage message) - { - ReadOnlySpan data = message.NotificationData.Span; - for (int i = 0; i < data.Length; i++) - { - if (data[i].TryGetValue(out DataChangeNotification? dcn)) - { - ReadOnlySpan items = - dcn.MonitoredItems.Span; - for (int j = 0; j < items.Length; j++) - { - (items[j] as IPooledEncodeable)?.Reuse(); - } - (dcn as IPooledEncodeable)?.Reuse(); - } - else if (data[i].TryGetValue(out EventNotificationList? enl)) - { - ReadOnlySpan events = enl.Events.Span; - for (int j = 0; j < events.Length; j++) - { - (events[j] as IPooledEncodeable)?.Reuse(); - } - (enl as IPooledEncodeable)?.Reuse(); - } - } - (message as IPooledEncodeable)?.Reuse(); - } } } diff --git a/Libraries/Opc.Ua.Server/Subscription/SubscriptionManager.cs b/Libraries/Opc.Ua.Server/Subscription/SubscriptionManager.cs index ef6248edab..a888040383 100644 --- a/Libraries/Opc.Ua.Server/Subscription/SubscriptionManager.cs +++ b/Libraries/Opc.Ua.Server/Subscription/SubscriptionManager.cs @@ -316,12 +316,12 @@ await StoreSubscriptionsAsync(cancellationToken) /// /// Stores durable subscriptions to be able to restore them after a restart /// - public virtual ValueTask StoreSubscriptionsAsync(CancellationToken cancellationToken = default) + public virtual async ValueTask StoreSubscriptionsAsync(CancellationToken cancellationToken = default) { // only store subscriptions if durable subscriptions are enabled if (!m_durableSubscriptionsEnabled || m_subscriptionStore == null) { - return default; + return; } var subscriptionsToStore = new List(); @@ -337,12 +337,14 @@ public virtual ValueTask StoreSubscriptionsAsync(CancellationToken cancellationT if (subscriptionsToStore.Count == 0) { - return default; + return; } try { - if (m_subscriptionStore.StoreSubscriptions(subscriptionsToStore)) + if (await m_subscriptionStore + .StoreSubscriptionsAsync(subscriptionsToStore, cancellationToken) + .ConfigureAwait(false)) { m_logger.LogInformation("{Count} Subscriptions stored", subscriptionsToStore.Count); } @@ -351,7 +353,6 @@ public virtual ValueTask StoreSubscriptionsAsync(CancellationToken cancellationT { m_logger.LogError(ex, "Failed to store {Count} subscriptions", subscriptionsToStore.Count); } - return default; } /// @@ -376,7 +377,9 @@ public virtual async ValueTask RestoreSubscriptionsAsync(CancellationToken cance try { - restoreResult = m_subscriptionStore.RestoreSubscriptions(); + restoreResult = await m_subscriptionStore + .RestoreSubscriptionsAsync(cancellationToken) + .ConfigureAwait(false); } catch (Exception ex) { @@ -417,7 +420,9 @@ public virtual async ValueTask RestoreSubscriptionsAsync(CancellationToken cance m_lastSubscriptionId = restoreResult.Subscriptions.Max(s => s.Id); - m_subscriptionStore.OnSubscriptionRestoreComplete(createdSubscriptions); + await m_subscriptionStore + .OnSubscriptionRestoreCompleteAsync(createdSubscriptions, cancellationToken) + .ConfigureAwait(false); } /// diff --git a/Stack/Opc.Ua.Core/Redundancy/AesCbcHmacRecordProtector.cs b/Stack/Opc.Ua.Core/Redundancy/AesCbcHmacRecordProtector.cs new file mode 100644 index 0000000000..1ca4810d31 --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/AesCbcHmacRecordProtector.cs @@ -0,0 +1,190 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; + +namespace Opc.Ua.Redundancy +{ + /// + /// Authenticated-encryption using + /// AES-256-CBC with HMAC-SHA256 in Encrypt-then-MAC construction. The MAC + /// is verified before any decryption (no padding-oracle exposure), so a + /// tampered or forged record is rejected fail-closed. The envelope is + /// [version:1][keyId:4 LE][IV:16][ciphertext][HMAC:32]; the MAC + /// covers the header + ciphertext. Distinct AES and MAC subkeys are + /// derived from the supplied master key. Cross-target-framework safe + /// (no AES-GCM dependency). + /// + public sealed class AesCbcHmacRecordProtector : IRecordProtector, IDisposable + { + /// + /// Creates a protector from a master key (≥ 32 bytes) and a key + /// identifier used for staged key rotation. + /// + /// The master key (at least 32 bytes). + /// + /// Identifies the key version; only records carrying the same id are + /// accepted by . + /// + public AesCbcHmacRecordProtector(ReadOnlySpan masterKey, uint keyId = 1) + { + if (masterKey.Length < MinMasterKeyLength) + { + throw new ArgumentException( + $"Master key must be at least {MinMasterKeyLength} bytes.", nameof(masterKey)); + } + m_keyId = keyId; + byte[] master = masterKey.ToArray(); + try + { + m_aesKey = DeriveKey(master, "OpcUaDistributed-AES256-CBC"); + m_macKey = DeriveKey(master, "OpcUaDistributed-HMAC-SHA256"); + } + finally + { + CryptoUtils.ZeroMemory(master); + } + } + + /// + public ByteString Protect(ByteString plaintext) + { + byte[] data = plaintext.IsNull ? Array.Empty() : plaintext.ToArray(); + + byte[] cipher; + byte[] iv; + using (var aes = Aes.Create()) + { + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.PKCS7; + aes.Key = m_aesKey; + // Let the algorithm generate a fresh random IV per record; do + // not assign one explicitly (CA5401). The IV is authenticated + // by the MAC below, so it is safe to ship in the clear. + aes.GenerateIV(); + iv = aes.IV; + using ICryptoTransform encryptor = aes.CreateEncryptor(); + cipher = encryptor.TransformFinalBlock(data, 0, data.Length); + } + + int headerLength = HeaderLength; + byte[] envelope = new byte[headerLength + cipher.Length + TagLength]; + envelope[0] = Version; + BinaryPrimitives.WriteUInt32LittleEndian(envelope.AsSpan(1, 4), m_keyId); + Buffer.BlockCopy(iv, 0, envelope, 5, IvLength); + Buffer.BlockCopy(cipher, 0, envelope, headerLength, cipher.Length); + + byte[] tag = ComputeTag(envelope, headerLength + cipher.Length); + Buffer.BlockCopy(tag, 0, envelope, headerLength + cipher.Length, TagLength); + return new ByteString(envelope); + } + + /// + public bool TryUnprotect(ByteString protectedRecord, out ByteString plaintext) + { + plaintext = default; + if (protectedRecord.IsNull) + { + return false; + } + + byte[] envelope = protectedRecord.ToArray(); + int headerLength = HeaderLength; + if (envelope.Length < headerLength + TagLength || envelope[0] != Version) + { + return false; + } + if (BinaryPrimitives.ReadUInt32LittleEndian(envelope.AsSpan(1, 4)) != m_keyId) + { + return false; + } + + int cipherLength = envelope.Length - headerLength - TagLength; + + // Verify the MAC before decrypting (Encrypt-then-MAC). + byte[] expectedTag = ComputeTag(envelope, headerLength + cipherLength); + var actualTag = new ReadOnlySpan(envelope, headerLength + cipherLength, TagLength); + if (!CryptoUtils.FixedTimeEquals(expectedTag, actualTag)) + { + return false; + } + + byte[] iv = new byte[IvLength]; + Buffer.BlockCopy(envelope, 5, iv, 0, IvLength); + + byte[] data; + using (var aes = Aes.Create()) + { + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.PKCS7; + aes.Key = m_aesKey; + aes.IV = iv; + using ICryptoTransform decryptor = aes.CreateDecryptor(); + data = decryptor.TransformFinalBlock(envelope, headerLength, cipherLength); + } + + plaintext = new ByteString(data); + return true; + } + + /// + /// Zeroizes the derived key material. + /// + public void Dispose() + { + CryptoUtils.ZeroMemory(m_aesKey); + CryptoUtils.ZeroMemory(m_macKey); + } + + private byte[] ComputeTag(byte[] buffer, int length) + { + using var hmac = new HMACSHA256(m_macKey); + return hmac.ComputeHash(buffer, 0, length); + } + + private static byte[] DeriveKey(byte[] masterKey, string label) + { + using var hmac = new HMACSHA256(masterKey); + return hmac.ComputeHash(Encoding.ASCII.GetBytes(label)); + } + + private const byte Version = 1; + private const int IvLength = 16; + private const int TagLength = 32; + private const int HeaderLength = 1 + 4 + IvLength; + private const int MinMasterKeyLength = 32; + + private readonly uint m_keyId; + private readonly byte[] m_aesKey; + private readonly byte[] m_macKey; + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/ILeaderElection.cs b/Stack/Opc.Ua.Core/Redundancy/ILeaderElection.cs new file mode 100644 index 0000000000..034059445f --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/ILeaderElection.cs @@ -0,0 +1,69 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Determines whether this replica is the writer (leader) in an + /// active/passive or single-writer active/active deployment. Supply the + /// predicate to an address-space synchronizer so + /// only the leader writes to the shared store ("shared read, master + /// write"), and to an IServiceLevelProvider so the leader advertises + /// the highest OPC UA ServiceLevel. + /// + public interface ILeaderElection : IAsyncDisposable + { + /// + /// true when this replica currently holds leadership. + /// + bool IsLeader { get; } + + /// + /// Raised when leadership is gained (true) or lost + /// (false). + /// + event Action? LeadershipChanged; + + /// + /// Attempts to acquire or renew leadership once and returns the + /// resulting leadership state. Safe to call repeatedly. + /// + /// Cancellation token. + ValueTask TryAcquireOrRenewAsync(CancellationToken ct = default); + + /// + /// Starts the background acquire/renew loop (if any). + /// + void Start(); + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/IRecordProtector.cs b/Stack/Opc.Ua.Core/Redundancy/IRecordProtector.cs new file mode 100644 index 0000000000..59bb8f82a4 --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/IRecordProtector.cs @@ -0,0 +1,59 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy +{ + /// + /// Protects extension state written to a shared store with authenticated encryption. + /// + /// + /// OPC 10000-4 §6.6 defines redundant-server behaviour but not a shared-store protection format. The shared store + /// is treated as an untrusted conduit: persisted records are encrypted and authenticated, and a tampered or forged + /// record fails closed so it is never decrypted or applied. External shared stores used for mirrored sessions, + /// subscriptions, retransmission queues, continuation points, or CRDT session records require a protector. + /// + public interface IRecordProtector + { + /// + /// Encrypts and authenticates , returning a + /// self-describing protected envelope. + /// + /// The record to protect. + ByteString Protect(ByteString plaintext); + + /// + /// Verifies and decrypts a protected envelope. Returns false + /// (fail-closed) when the record is missing its envelope, fails the + /// integrity check, or was produced under a different key. + /// + /// The protected envelope. + /// The recovered plaintext on success. + bool TryUnprotect(ByteString protectedRecord, out ByteString plaintext); + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/ISharedKeyValueStore.cs b/Stack/Opc.Ua.Core/Redundancy/ISharedKeyValueStore.cs new file mode 100644 index 0000000000..1a461ba5dd --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/ISharedKeyValueStore.cs @@ -0,0 +1,118 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: minimal shared key/value backend used to replicate state across a + /// RedundantServerSet. + /// + /// + /// + /// OPC 10000-4 §6.6 standardizes redundancy behaviour, discovery data, ServiceLevel selection, and Failover + /// actions; it does not define a shared storage protocol. This value-add extension is the lowest-level abstraction + /// used by the distributed AddressSpace, session, subscription, continuation-point, nonce, and lease mirrors. + /// + /// + /// Keys are opaque, ordinal strings. Values are + /// payloads. Implementations must be safe for concurrent calls. External shared stores must be paired with an + /// for mirrored records that contain secrets or notifications. + /// + /// + public interface ISharedKeyValueStore + { + /// + /// Reads the value stored under . + /// + /// The key to read. + /// Cancellation token. + /// + /// Found = true and the stored value when the key exists; + /// otherwise Found = false and a null + /// . + /// + ValueTask<(bool Found, ByteString Value)> TryGetAsync(string key, CancellationToken ct = default); + + /// + /// Unconditionally writes under + /// . + /// + /// The key to write. + /// The value to store. + /// Cancellation token. + ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default); + + /// + /// Atomically writes under + /// only when the current value matches + /// . A null + /// () requires the key to be absent. + /// This is the "master write" / single-writer primitive used by the + /// leader-election layer. + /// + /// The key to write. + /// + /// The value the key is expected to currently hold, or a null + /// to require absence. + /// + /// The value to store on success. + /// Cancellation token. + /// true when the swap succeeded. + ValueTask CompareAndSwapAsync(string key, ByteString expected, ByteString value, CancellationToken ct = default); + + /// + /// Removes . + /// + /// The key to remove. + /// Cancellation token. + /// true when a value was removed. + ValueTask DeleteAsync(string key, CancellationToken ct = default); + + /// + /// Enumerates a snapshot of every key/value pair whose key starts + /// with . Used for hydration. + /// + /// The key prefix to match (may be empty). + /// Cancellation token. + IAsyncEnumerable> ScanAsync(string keyPrefix, CancellationToken ct = default); + + /// + /// Streams changes for every key that starts with + /// until is + /// cancelled. Only changes that occur after the call are observed. + /// + /// The key prefix to match (may be empty). + /// Cancellation token that stops the watch. + IAsyncEnumerable WatchAsync(string keyPrefix, CancellationToken ct = default); + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/InMemorySharedKeyValueStore.cs b/Stack/Opc.Ua.Core/Redundancy/InMemorySharedKeyValueStore.cs new file mode 100644 index 0000000000..ec3e00f3f5 --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/InMemorySharedKeyValueStore.cs @@ -0,0 +1,237 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: in-process, thread-safe . A single + /// instance shared by multiple node managers (or multiple in-process + /// server instances) provides a deterministic backend for both + /// active/active and active/passive testing and single-process + /// deployments. Redis / external stores are drop-in replacements of the + /// same contract. + /// + public sealed class InMemorySharedKeyValueStore : ISharedKeyValueStore, IDisposable + { + /// + /// Reads the value stored under . + /// + /// + public ValueTask<(bool Found, ByteString Value)> TryGetAsync(string key, CancellationToken ct = default) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + + lock (m_lock) + { + if (m_data.TryGetValue(key, out ByteString value)) + { + return new ValueTask<(bool, ByteString)>((true, value)); + } + } + return new ValueTask<(bool, ByteString)>((false, default)); + } + + /// + public ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + + lock (m_lock) + { + m_data[key] = value; + PublishLocked(new KeyValueChange { Kind = KeyValueChangeKind.Set, Key = key, Value = value }); + } + return default; + } + + /// + public ValueTask CompareAndSwapAsync( + string key, + ByteString expected, + ByteString value, + CancellationToken ct = default) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + + lock (m_lock) + { + bool present = m_data.TryGetValue(key, out ByteString current); + bool matches = expected.IsNull ? !present : present && current.Equals(expected); + if (!matches) + { + return new ValueTask(false); + } + + m_data[key] = value; + PublishLocked(new KeyValueChange { Kind = KeyValueChangeKind.Set, Key = key, Value = value }); + return new ValueTask(true); + } + } + + /// + public ValueTask DeleteAsync(string key, CancellationToken ct = default) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + + lock (m_lock) + { + if (m_data.Remove(key)) + { + PublishLocked(new KeyValueChange { Kind = KeyValueChangeKind.Delete, Key = key }); + return new ValueTask(true); + } + } + return new ValueTask(false); + } + + /// + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + keyPrefix ??= string.Empty; + + List> snapshot; + lock (m_lock) + { + snapshot = new List>(m_data.Count); + foreach (KeyValuePair entry in m_data) + { + if (entry.Key.StartsWith(keyPrefix, StringComparison.Ordinal)) + { + snapshot.Add(entry); + } + } + } + + foreach (KeyValuePair entry in snapshot) + { + ct.ThrowIfCancellationRequested(); + yield return entry; + } + + await Task.CompletedTask.ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable WatchAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + var watcher = new Watcher(keyPrefix ?? string.Empty); + lock (m_lock) + { + m_watchers.Add(watcher); + } + + try + { + await foreach (KeyValueChange change in watcher.Channel.Reader + .ReadAllAsync(ct) + .ConfigureAwait(false)) + { + yield return change; + } + } + finally + { + lock (m_lock) + { + m_watchers.Remove(watcher); + } + watcher.Channel.Writer.TryComplete(); + } + } + + /// + /// Completes all outstanding watchers. + /// + public void Dispose() + { + lock (m_lock) + { + foreach (Watcher watcher in m_watchers) + { + watcher.Channel.Writer.TryComplete(); + } + m_watchers.Clear(); + m_data.Clear(); + } + } + + private void PublishLocked(KeyValueChange change) + { + for (int ii = 0; ii < m_watchers.Count; ii++) + { + Watcher watcher = m_watchers[ii]; + if (change.Key.StartsWith(watcher.Prefix, StringComparison.Ordinal)) + { + watcher.Channel.Writer.TryWrite(change); + } + } + } + + private sealed class Watcher + { + public Watcher(string prefix) + { + Prefix = prefix; + } + + public string Prefix { get; } + + public Channel Channel { get; } = + System.Threading.Channels.Channel.CreateUnbounded( + new UnboundedChannelOptions { SingleReader = true, SingleWriter = false }); + } + + private readonly Lock m_lock = new(); + private readonly Dictionary m_data = new(StringComparer.Ordinal); + private readonly List m_watchers = []; + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/KeyRingRecordProtector.cs b/Stack/Opc.Ua.Core/Redundancy/KeyRingRecordProtector.cs new file mode 100644 index 0000000000..763c3df06a --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/KeyRingRecordProtector.cs @@ -0,0 +1,117 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; + +namespace Opc.Ua.Redundancy +{ + /// + /// An that supports staged, zero-downtime + /// key rotation. New records are always written (and authenticated) with a + /// single active key, while reads are verified against the active + /// key first and then any number of retired keys still present in + /// the store. This lets an operator introduce a new key fleet-wide, let + /// records re-write under it over time, and only then drop the old key — + /// without a flag-day re-encryption (staged key + /// rotation). Each member key is identified by its keyId, so a record + /// is only ever decrypted by the key version that produced it. + /// + public sealed class KeyRingRecordProtector : IRecordProtector, IDisposable + { + /// + /// Creates a key ring. + /// + /// + /// The active protector used to protect (write) every new record. + /// + /// + /// Older protectors retained only to unprotect (read) records written + /// before the most recent rotation. May be empty. + /// + public KeyRingRecordProtector(IRecordProtector active, params IRecordProtector[] retired) + { + m_active = active ?? throw new ArgumentNullException(nameof(active)); + var all = new List(1 + (retired?.Length ?? 0)) { active }; + if (retired != null) + { + foreach (IRecordProtector protector in retired) + { + if (protector == null) + { + throw new ArgumentException( + "Retired protectors must not be null.", nameof(retired)); + } + all.Add(protector); + } + } + m_all = all; + } + + /// + public ByteString Protect(ByteString plaintext) + { + return m_active.Protect(plaintext); + } + + /// + public bool TryUnprotect(ByteString protectedRecord, out ByteString plaintext) + { + // A record carries the key-id of the key that produced it; each + // member rejects (fail-closed) any record it did not produce, so the + // first success is unambiguous. + foreach (IRecordProtector protector in m_all) + { + if (protector.TryUnprotect(protectedRecord, out plaintext)) + { + return true; + } + } + plaintext = default; + return false; + } + + /// + /// Disposes every member protector that owns key material. + /// + public void Dispose() + { + foreach (IRecordProtector protector in m_all) + { + if (protector is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + + private readonly IRecordProtector m_active; + private readonly IReadOnlyList m_all; + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/KeyValueChange.cs b/Stack/Opc.Ua.Core/Redundancy/KeyValueChange.cs new file mode 100644 index 0000000000..27e4db439f --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/KeyValueChange.cs @@ -0,0 +1,70 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: the kind of mutation reported by a . + /// + public enum KeyValueChangeKind + { + /// + /// The key was created or updated to a new value. + /// + Set, + + /// + /// The key was removed. + /// + Delete + } + + /// + /// Extension beyond OPC 10000-4 §6.6: a single change observed on an + /// change-feed (). + /// + public sealed record KeyValueChange + { + /// + /// The kind of mutation. + /// + public KeyValueChangeKind Kind { get; init; } + + /// + /// The affected key. + /// + public string Key { get; init; } = string.Empty; + + /// + /// The new value for changes; + /// a null for deletions. + /// + public ByteString Value { get; init; } + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/NullRecordProtector.cs b/Stack/Opc.Ua.Core/Redundancy/NullRecordProtector.cs new file mode 100644 index 0000000000..7564c6fa31 --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/NullRecordProtector.cs @@ -0,0 +1,60 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua.Redundancy +{ + /// + /// A no-op that passes records through + /// unchanged. It is the default for single-process / in-memory demos and + /// tests. Production deployments backed by a network store (e.g. Redis) + /// must configure an authenticated-encryption protector + /// (); see + /// Docs/HighAvailability.md. + /// + public sealed class NullRecordProtector : IRecordProtector + { + /// + /// The shared singleton instance. + /// + public static NullRecordProtector Instance { get; } = new(); + + /// + public ByteString Protect(ByteString plaintext) + { + return plaintext; + } + + /// + public bool TryUnprotect(ByteString protectedRecord, out ByteString plaintext) + { + plaintext = protectedRecord; + return true; + } + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/SharedStoreLeaseElection.cs b/Stack/Opc.Ua.Core/Redundancy/SharedStoreLeaseElection.cs new file mode 100644 index 0000000000..8d73c517f3 --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/SharedStoreLeaseElection.cs @@ -0,0 +1,281 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Buffers.Binary; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Opc.Ua.Redundancy +{ + /// + /// Extension beyond OPC 10000-4 §6.6: lease-based over an + /// . A single lease key holds the + /// current leader's id and an expiry; leadership is acquired and renewed + /// atomically with + /// ("shared read, master write"). A leader that stops renewing loses the + /// lease once it expires, allowing a standby to take over. + /// + public sealed class SharedStoreLeaseElection : ILeaderElection + { + /// + /// Creates a lease election. + /// + /// The shared key/value backend. + /// The key holding the lease. + /// This replica's unique identity. + /// + /// How long an acquired lease remains valid without renewal. + /// + /// + /// How often the background loop renews the lease. + /// + /// Time source (defaults to system). + /// Optional logger. + public SharedStoreLeaseElection( + ISharedKeyValueStore store, + string leaseKey, + string nodeId, + TimeSpan leaseDuration, + TimeSpan renewInterval, + TimeProvider? timeProvider = null, + ILogger? logger = null) + { + m_store = store ?? throw new ArgumentNullException(nameof(store)); + if (string.IsNullOrEmpty(leaseKey)) + { + throw new ArgumentException("Lease key must not be null or empty.", nameof(leaseKey)); + } + if (string.IsNullOrEmpty(nodeId)) + { + throw new ArgumentException("Node id must not be null or empty.", nameof(nodeId)); + } + m_leaseKey = leaseKey; + m_nodeId = nodeId; + m_leaseDuration = leaseDuration; + m_renewInterval = renewInterval; + m_timeProvider = timeProvider ?? TimeProvider.System; + m_logger = logger; + } + + /// + public bool IsLeader + { + get + { + lock (m_lock) + { + return m_isLeader; + } + } + } + + /// + public event Action? LeadershipChanged; + + /// + public async ValueTask TryAcquireOrRenewAsync(CancellationToken ct = default) + { + (bool found, ByteString current) = await m_store.TryGetAsync(m_leaseKey, ct).ConfigureAwait(false); + long nowTicks = m_timeProvider.GetUtcNow().UtcTicks; + + bool canTake = !found; + if (found) + { + canTake = !TryParseLease(current, out string owner, out long expiryTicks) + || nowTicks >= expiryTicks + || string.Equals(owner, m_nodeId, StringComparison.Ordinal); + } + + if (!canTake) + { + SetLeader(false); + return false; + } + + ByteString newLease = EncodeLease(m_nodeId, nowTicks + m_leaseDuration.Ticks); + ByteString expected = found ? current : default; + bool acquired = await m_store + .CompareAndSwapAsync(m_leaseKey, expected, newLease, ct) + .ConfigureAwait(false); + SetLeader(acquired); + return acquired; + } + + /// + public void Start() + { + lock (m_lock) + { + if (m_started) + { + return; + } + m_started = true; + m_loop = Task.Run(() => RenewLoopAsync(m_cts.Token)); + } + } + + /// + public async ValueTask DisposeAsync() + { + lock (m_lock) + { + if (m_disposed) + { + return; + } + m_disposed = true; + } + + m_cts.Cancel(); + if (m_loop != null) + { + try + { + await m_loop.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // expected on shutdown + } + } + + await ReleaseIfOwnedAsync().ConfigureAwait(false); + m_cts.Dispose(); + } + + private async Task RenewLoopAsync(CancellationToken ct) + { + try + { + while (!ct.IsCancellationRequested) + { + try + { + await TryAcquireOrRenewAsync(ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + m_logger?.LogError(ex, "Lease election renew failed for {NodeId}.", m_nodeId); + } + + await Task.Delay(m_renewInterval, ct).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // shutdown + } + } + + private async Task ReleaseIfOwnedAsync() + { + try + { + (bool found, ByteString current) = await m_store + .TryGetAsync(m_leaseKey, CancellationToken.None) + .ConfigureAwait(false); + if (found && + TryParseLease(current, out string owner, out _) && + string.Equals(owner, m_nodeId, StringComparison.Ordinal)) + { + await m_store.DeleteAsync(m_leaseKey, CancellationToken.None).ConfigureAwait(false); + } + } + catch (Exception ex) + { + m_logger?.LogError(ex, "Lease election release failed for {NodeId}.", m_nodeId); + } + } + + private void SetLeader(bool value) + { + bool changed; + lock (m_lock) + { + changed = m_isLeader != value; + m_isLeader = value; + } + if (changed) + { + LeadershipChanged?.Invoke(value); + } + } + + private static ByteString EncodeLease(string owner, long expiryUtcTicks) + { + byte[] ownerBytes = Encoding.UTF8.GetBytes(owner); + byte[] buffer = new byte[4 + ownerBytes.Length + 8]; + BinaryPrimitives.WriteInt32LittleEndian(buffer.AsSpan(0, 4), ownerBytes.Length); + ownerBytes.CopyTo(buffer, 4); + BinaryPrimitives.WriteInt64LittleEndian(buffer.AsSpan(4 + ownerBytes.Length, 8), expiryUtcTicks); + return new ByteString(buffer); + } + + private static bool TryParseLease(ByteString raw, out string owner, out long expiryUtcTicks) + { + owner = string.Empty; + expiryUtcTicks = 0; + byte[] bytes = raw.ToArray(); + if (bytes.Length < 4) + { + return false; + } + int ownerLength = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(0, 4)); + if (ownerLength < 0 || bytes.Length < 4 + ownerLength + 8) + { + return false; + } + owner = Encoding.UTF8.GetString(bytes, 4, ownerLength); + expiryUtcTicks = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(4 + ownerLength, 8)); + return true; + } + + private readonly ISharedKeyValueStore m_store; + private readonly string m_leaseKey; + private readonly string m_nodeId; + private readonly TimeSpan m_leaseDuration; + private readonly TimeSpan m_renewInterval; + private readonly TimeProvider m_timeProvider; + private readonly ILogger? m_logger; + private readonly Lock m_lock = new(); + private readonly CancellationTokenSource m_cts = new(); + private Task? m_loop; + private bool m_isLeader; + private bool m_started; + private bool m_disposed; + } +} diff --git a/Stack/Opc.Ua.Core/Redundancy/StaticLeaderElection.cs b/Stack/Opc.Ua.Core/Redundancy/StaticLeaderElection.cs new file mode 100644 index 0000000000..726d86161d --- /dev/null +++ b/Stack/Opc.Ua.Core/Redundancy/StaticLeaderElection.cs @@ -0,0 +1,76 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Opc.Ua.Redundancy +{ + /// + /// A fixed-role for static deployments + /// (single instance, or an externally-assigned leader). Use + /// for dynamic election. + /// + public sealed class StaticLeaderElection : ILeaderElection + { + /// + /// Creates a static election with a fixed leadership state. + /// + /// Whether this replica is the leader. + public StaticLeaderElection(bool isLeader) + { + IsLeader = isLeader; + } + + /// + public bool IsLeader { get; } + + /// + public event Action? LeadershipChanged; + + /// + public ValueTask TryAcquireOrRenewAsync(CancellationToken ct = default) + { + return new ValueTask(IsLeader); + } + + /// + public void Start() + { + LeadershipChanged?.Invoke(IsLeader); + } + + /// + public ValueTask DisposeAsync() + { + return default; + } + } +} diff --git a/Stack/Opc.Ua.Core/Security/Certificates/EncryptedSecret.cs b/Stack/Opc.Ua.Core/Security/Certificates/EncryptedSecret.cs index 6045aab604..a4471a1ade 100644 --- a/Stack/Opc.Ua.Core/Security/Certificates/EncryptedSecret.cs +++ b/Stack/Opc.Ua.Core/Security/Certificates/EncryptedSecret.cs @@ -1294,11 +1294,8 @@ private static void ZeroMemory(byte[]? buffer) { return; } -#if NET8_0_OR_GREATER - CryptographicOperations.ZeroMemory(buffer); -#else - Array.Clear(buffer, 0, buffer.Length); -#endif + + CryptoUtils.ZeroMemory(buffer); } } } diff --git a/Stack/Opc.Ua.Core/Stack/Types/ServiceLevel.cs b/Stack/Opc.Ua.Core/Stack/Types/ServiceLevel.cs new file mode 100644 index 0000000000..b9b3eaa64f --- /dev/null +++ b/Stack/Opc.Ua.Core/Stack/Types/ServiceLevel.cs @@ -0,0 +1,166 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +namespace Opc.Ua +{ + /// + /// Identifies the OPC UA ServiceLevel subrange defined by OPC 10000-4 §6.6.2.4.2 Table 105. + /// + public enum ServiceLevelSubrange + { + /// + /// The server is in maintenance and clients should not connect. + /// + Maintenance, + + /// + /// The server is not currently providing process data. + /// + NoData, + + /// + /// The server is operational with degraded service. + /// + Degraded, + + /// + /// The server is healthy and operational. + /// + Healthy + } + + /// + /// Defines OPC UA ServiceLevel subranges from OPC 10000-4 §6.6.2.4.2 Table 105. + /// + public static class ServiceLevels + { + /// + /// The server is in maintenance and clients should not connect. + /// + public const byte Maintenance = 0; + + /// + /// The server is not currently providing process data. + /// + public const byte NoData = 1; + + /// + /// The lowest degraded-but-operational service level. + /// + public const byte DegradedMinimum = 2; + + /// + /// The highest degraded-but-operational service level. + /// + public const byte DegradedMaximum = 199; + + /// + /// The lowest healthy service level. + /// + public const byte HealthyMinimum = 200; + + /// + /// The highest service level. + /// + public const byte Maximum = 255; + + /// + /// Gets the subrange that contains the service level. + /// + /// The service level. + /// The matching subrange. + public static ServiceLevelSubrange GetSubrange(byte level) + { + if (level == Maintenance) + { + return ServiceLevelSubrange.Maintenance; + } + if (level == NoData) + { + return ServiceLevelSubrange.NoData; + } + if (level < HealthyMinimum) + { + return ServiceLevelSubrange.Degraded; + } + + return ServiceLevelSubrange.Healthy; + } + + /// + /// Returns whether the service level is in the healthy subrange. + /// + /// The service level. + /// true if the server is healthy. + public static bool IsHealthy(byte level) + { + return GetSubrange(level) == ServiceLevelSubrange.Healthy; + } + + /// + /// Returns whether the service level is in the degraded subrange. + /// + /// The service level. + /// true if the server is degraded but operational. + public static bool IsDegraded(byte level) + { + return GetSubrange(level) == ServiceLevelSubrange.Degraded; + } + + /// + /// Returns whether the service level reports no process data. + /// + /// The service level. + /// true if the server has no data. + public static bool IsNoData(byte level) + { + return level == NoData; + } + + /// + /// Returns whether the service level reports maintenance. + /// + /// The service level. + /// true if the server is in maintenance. + public static bool IsMaintenance(byte level) + { + return level == Maintenance; + } + + /// + /// Returns whether the service level is operational. + /// + /// The service level. + /// true if the server is degraded or healthy. + public static bool IsOperational(byte level) + { + return IsDegraded(level) || IsHealthy(level); + } + } +} diff --git a/Tests/Opc.Ua.Aot.Tests/CrdtAotTests.cs b/Tests/Opc.Ua.Aot.Tests/CrdtAotTests.cs new file mode 100644 index 0000000000..84fbfc5f6c --- /dev/null +++ b/Tests/Opc.Ua.Aot.Tests/CrdtAotTests.cs @@ -0,0 +1,84 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using Opc.Ua.Redundancy.Server; + +// CA2007: AOT tests run without a SynchronizationContext. +#pragma warning disable CA2007 + +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Aot.Tests +{ + /// + /// AOT integration tests that exercise the CRDT active/active building + /// blocks, ensuring they are reachable and functional under NativeAOT. + /// + public class CrdtAotTests + { + [Test] + public async Task CrdtKeyValueStoreRoundTripsUnderAotAsync() + { + await using var network = new InMemoryNetwork(); + await using var store = new CrdtSharedKeyValueStore( + ReplicaId.FromUInt64(1), + network.CreateTransport(), + TimeProvider.System, + CrdtReaderOptions.Default); + + var value = new ByteString(new byte[] { 10, 20, 30 }); + await store.SetAsync("session/aot", value); + + (bool found, ByteString stored) = await store.TryGetAsync("session/aot"); + + await Assert.That(found).IsTrue(); + byte[] bytes = stored.ToArray(); + await Assert.That(bytes.Length).IsEqualTo(3); + await Assert.That(bytes[0]).IsEqualTo((byte)10); + await Assert.That(bytes[1]).IsEqualTo((byte)20); + await Assert.That(bytes[2]).IsEqualTo((byte)30); + } + + [Test] + public async Task CrdtOptionsConfigureTransportUnderAotAsync() + { + var options = new ReplicatedAddressSpaceOptions + { + ReplicaId = ReplicaId.New() + }; + options.UseUdpGossip(System.Net.IPAddress.Loopback, 0); + + await Assert.That(options.TransportFactory).IsNotNull(); + } + } +} diff --git a/Tests/Opc.Ua.Aot.Tests/DistributedRedundancyAotTests.cs b/Tests/Opc.Ua.Aot.Tests/DistributedRedundancyAotTests.cs new file mode 100644 index 0000000000..a626289d77 --- /dev/null +++ b/Tests/Opc.Ua.Aot.Tests/DistributedRedundancyAotTests.cs @@ -0,0 +1,303 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Opc.Ua.Server; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy.K8s; +using Opc.Ua.Redundancy; + +// CA2007: AOT tests run without a SynchronizationContext. +#pragma warning disable CA2007 + +namespace Opc.Ua.Aot.Tests +{ + /// + /// NativeAOT exercises for distributed redundancy serialization paths. + /// + public class DistributedRedundancyAotTests + { + [Test] + public async Task KubernetesSourceGeneratedJsonRoundTripsLeaseAndEndpointSliceAsync() + { + var lease = new KubernetesLease + { + Metadata = new KubernetesObjectMetadata + { + Name = "opcua-leader", + Namespace = "default", + ResourceVersion = "12" + }, + Spec = new KubernetesLeaseSpec + { + HolderIdentity = "replica-a", + LeaseDurationSeconds = 30, + AcquireTime = "2026-06-27T08:00:00Z", + RenewTime = "2026-06-27T08:00:10Z", + LeaseTransitions = 2 + } + }; + string leaseJson = JsonSerializer.Serialize( + lease, + KubernetesJsonContext.Default.KubernetesLease); + KubernetesLease? decodedLease = JsonSerializer.Deserialize( + leaseJson, + KubernetesJsonContext.Default.KubernetesLease); + + await Assert.That(decodedLease).IsNotNull(); + await Assert.That(decodedLease!.Spec.HolderIdentity).IsEqualTo("replica-a"); + + var slices = new KubernetesEndpointSliceList + { + Items = + [ + new KubernetesEndpointSlice + { + Endpoints = + [ + new KubernetesEndpoint + { + Addresses = ["10.0.0.11"], + Conditions = new KubernetesEndpointConditions { Ready = true } + } + ], + Ports = [new KubernetesEndpointPort { Name = "opcua-tcp", Port = 4840 }] + } + ] + }; + string sliceJson = JsonSerializer.Serialize( + slices, + KubernetesJsonContext.Default.KubernetesEndpointSliceList); + KubernetesEndpointSliceList? decodedSlices = JsonSerializer.Deserialize( + sliceJson, + KubernetesJsonContext.Default.KubernetesEndpointSliceList); + + await Assert.That(decodedSlices).IsNotNull(); + await Assert.That(decodedSlices!.Items[0].Endpoints[0].Addresses[0]).IsEqualTo("10.0.0.11"); + } + + [Test] + public async Task KubernetesLeaseAndPeerDiscoveryUseMockedInClusterClientAsync() + { + var client = new FakeKubernetesApiClient(); + var leaderOptions = new KubernetesLeaderElectionOptions(); + leaderOptions.Kubernetes.Namespace = "default"; + leaderOptions.Kubernetes.NodeId = "replica-a"; + leaderOptions.LeaseName = "opcua-leader"; + var election = new KubernetesLeaseLeaderElection(client, leaderOptions); + + bool acquired = await election.TryAcquireOrRenewAsync(); + + await Assert.That(acquired).IsTrue(); + await Assert.That(client.Lease).IsNotNull(); + await Assert.That(client.Lease!.Spec.HolderIdentity).IsEqualTo("replica-a"); + + var discoveryOptions = new KubernetesPeerDiscoveryOptions(); + discoveryOptions.Kubernetes.Namespace = "default"; + discoveryOptions.ServiceName = "opcua"; + discoveryOptions.LocalAddress = "10.0.0.10"; + client.EndpointSlices = new KubernetesEndpointSliceList + { + Items = + [ + new KubernetesEndpointSlice + { + Endpoints = + [ + new KubernetesEndpoint + { + Addresses = ["10.0.0.10", "10.0.0.11"], + Conditions = new KubernetesEndpointConditions { Ready = true } + }, + new KubernetesEndpoint + { + Addresses = ["10.0.0.12"], + Conditions = new KubernetesEndpointConditions { Ready = false } + } + ], + Ports = [new KubernetesEndpointPort { Name = "opcua-tcp", Port = 4840 }] + } + ] + }; + var discovery = new KubernetesPeerDiscovery(client, discoveryOptions); + + ArrayOf peers = await discovery.RefreshAsync(); + + await Assert.That(peers.Count).IsEqualTo(1); + await Assert.That(peers[0]).IsEqualTo("opc.tcp://10.0.0.11:4840"); + } + + [Test] + public async Task BaseMirrorStoresRoundTripProtectedSessionAndSubscriptionAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(31)); + ITelemetryContext telemetry = DefaultTelemetry.Create(builder => + builder.SetMinimumLevel(LogLevel.Warning)); + ServiceMessageContext context = ServiceMessageContext.CreateEmpty(telemetry); + var sessionStore = new SharedKeyValueSessionStore(kv, context, protector); + var token = new NodeId(Guid.NewGuid(), 1); + var entry = new SharedSessionEntry + { + SessionId = new NodeId(Guid.NewGuid(), 1), + AuthenticationToken = token, + SessionName = "aot-session", + CreatedAt = DateTime.UtcNow, + LastActivatedAt = DateTime.UtcNow, + ServerNonce = ByteString.From(new byte[] { 1, 2, 3 }), + ClientNonce = ByteString.From(new byte[] { 4, 5, 6 }), + SecurityPolicyUri = SecurityPolicies.None, + SecurityMode = (int)MessageSecurityMode.None, + EndpointUrl = "opc.tcp://localhost:4840", + SessionTimeout = 60000, + ClientDescription = new ApplicationDescription + { + ApplicationUri = "urn:aot-client", + ApplicationType = ApplicationType.Client + }, + SecretMaterial = ByteString.From(new byte[] { 7, 8, 9 }) + }; + + await sessionStore.PutAsync(entry); + SharedSessionEntry? restored = await sessionStore.TryGetAsync(token); + + await Assert.That(restored).IsNotNull(); + await Assert.That(restored!.SessionName).IsEqualTo("aot-session"); + + await using var subscriptionStore = new SharedKeyValueSubscriptionStore(kv, context, protector); + var subscription = new StoredSubscription + { + Id = 42, + IsDurable = true, + MaxLifetimeCount = 120, + MaxKeepaliveCount = 12, + MaxMessageCount = 8, + MaxNotificationsPerPublish = 16, + PublishingInterval = 250, + Priority = 7, + SentMessages = [], + UserIdentityToken = new AnonymousIdentityToken(), + MonitoredItems = + [ + new StoredMonitoredItem + { + AttributeId = Attributes.Value, + ClientHandle = 100, + Id = 500, + NodeId = new NodeId("Temperature", 2), + MonitoringMode = MonitoringMode.Reporting, + QueueSize = 10, + SamplingInterval = 100, + SubscriptionId = 42, + TimestampsToReturn = TimestampsToReturn.Both, + FilterToUse = new DataChangeFilter + { + Trigger = DataChangeTrigger.StatusValue, + DeadbandType = (uint)DeadbandType.Absolute, + DeadbandValue = 1 + } + } + ] + }; + + await subscriptionStore.StoreSubscriptionsAsync([subscription]); + RestoreSubscriptionResult result = await subscriptionStore.RestoreSubscriptionsAsync(); + List subscriptions = [.. result.Subscriptions]; + + await Assert.That(result.Success).IsTrue(); + await Assert.That(subscriptions.Count).IsEqualTo(1); + await Assert.That(subscriptions[0].Id).IsEqualTo(42u); + } + + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int ii = 0; ii < key.Length; ii++) + { + key[ii] = (byte)(seed + ii); + } + return key; + } + + private sealed class FakeKubernetesApiClient : IKubernetesApiClient + { + public bool IsInCluster => true; + + public KubernetesLease? Lease { get; set; } + + public KubernetesEndpointSliceList EndpointSlices { get; set; } = new(); + + public ValueTask GetLeaseAsync( + string namespaceName, + string name, + CancellationToken ct) + { + return new ValueTask(Lease); + } + + public ValueTask CreateLeaseAsync( + string namespaceName, + KubernetesLease lease, + CancellationToken ct) + { + Lease = lease; + return new ValueTask(lease); + } + + public ValueTask ReplaceLeaseAsync( + string namespaceName, + string name, + KubernetesLease lease, + CancellationToken ct) + { + Lease = lease; + return new ValueTask(lease); + } + + public ValueTask DeleteLeaseAsync(string namespaceName, string name, CancellationToken ct) + { + Lease = null; + return default; + } + + public ValueTask ListEndpointSlicesAsync( + string namespaceName, + string serviceName, + CancellationToken ct) + { + return new ValueTask(EndpointSlices); + } + } + } +} diff --git a/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj b/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj index 65904b2a35..8baf065598 100644 --- a/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj +++ b/Tests/Opc.Ua.Aot.Tests/Opc.Ua.Aot.Tests.csproj @@ -25,6 +25,9 @@ + + + diff --git a/Tests/Opc.Ua.Aot.Tests/RaftAotTests.cs b/Tests/Opc.Ua.Aot.Tests/RaftAotTests.cs new file mode 100644 index 0000000000..e556e84bd4 --- /dev/null +++ b/Tests/Opc.Ua.Aot.Tests/RaftAotTests.cs @@ -0,0 +1,72 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Threading.Tasks; +using Opc.Ua.Redundancy; + +// CA2007: AOT tests run without a SynchronizationContext. +#pragma warning disable CA2007 + +namespace Opc.Ua.Aot.Tests +{ + /// + /// AOT integration tests that exercise the Raft (strong-consistency) building blocks over a real + /// replica, ensuring the external RaftCs engine is reachable and functional under + /// NativeAOT. + /// + public class RaftAotTests + { + [Test] + public async Task RaftCsStoreCompareAndSwapUnderAotAsync() + { + await using RaftCsConsensus consensus = RaftCsConsensus.CreateSingleNode(); + await using var store = new RaftSharedKeyValueStore(consensus, ownsConsensus: false); + + var value = new ByteString(new byte[] { 7, 8, 9 }); + bool created = await store.CompareAndSwapAsync("raft/aot", default, value); + (bool found, ByteString stored) = await store.TryGetAsync("raft/aot"); + + await Assert.That(created).IsTrue(); + await Assert.That(found).IsTrue(); + await Assert.That(stored.ToArray().Length).IsEqualTo(3); + } + + [Test] + public async Task RaftLeaderElectionUnderAotAsync() + { + await using RaftCsConsensus consensus = RaftCsConsensus.CreateSingleNode(); + await using var election = new RaftLeaderElection(consensus); + + await consensus.StartAsync(); + + await Assert.That(consensus.IsLeader).IsTrue(); + await Assert.That(election.IsLeader).IsTrue(); + } + } +} diff --git a/Tests/Opc.Ua.Client.Tests/Session/ClientFailoverCoordinatorTests.cs b/Tests/Opc.Ua.Client.Tests/Session/ClientFailoverCoordinatorTests.cs new file mode 100644 index 0000000000..841cfdb3a8 --- /dev/null +++ b/Tests/Opc.Ua.Client.Tests/Session/ClientFailoverCoordinatorTests.cs @@ -0,0 +1,245 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; + +namespace Opc.Ua.Client.Tests.ManagedSession +{ + /// + /// Tests for . + /// + [TestFixture] + [Category("Client")] + [Category("ClientRedundancy")] + public sealed class ClientFailoverCoordinatorTests + { + [Test] + public async Task TransferActiveSubscriptionsUsesDiagnosticsAndTransferServiceAsync() + { + var sessionId = new NodeId(42); + Mock session = CreateSession("operator"); + SetupDiagnostics(session, sessionId, "active-client", [11u, 12u]); + ArrayOf transferredIds = []; + session.Setup(s => s.TransferSubscriptionsAsync( + null, + It.IsAny>(), + true, + It.IsAny())) + .Callback, bool, CancellationToken>( + (_, ids, _, _) => transferredIds = ids) + .ReturnsAsync(new TransferSubscriptionsResponse + { + Results = + [ + new TransferResult { StatusCode = StatusCodes.Good }, + new TransferResult { StatusCode = StatusCodes.Good } + ], + DiagnosticInfos = [] + }); + + var coordinator = new ClientFailoverCoordinator(); + ArrayOf results = await coordinator + .TransferActiveSubscriptionsAsync( + session.Object, + new ClientRedundancyTransferOptions + { + ActiveSessionName = "active-client", + ActiveUserDisplayName = "operator", + SendInitialValues = true + }) + .ConfigureAwait(false); + + Assert.That(results, Has.Count.EqualTo(2)); + Assert.That(transferredIds, Is.EqualTo(new uint[] { 11, 12 })); + } + + [Test] + public void TransferActiveSubscriptionsRejectsDifferentUser() + { + Mock session = CreateSession("backup-user"); + var coordinator = new ClientFailoverCoordinator(); + + Assert.ThrowsAsync( + async () => await coordinator.TransferActiveSubscriptionsAsync( + session.Object, + new ClientRedundancyTransferOptions + { + ActiveUserDisplayName = "active-user", + ActiveSessionName = "active-client" + }).ConfigureAwait(false)); + } + + [Test] + public async Task DiscoverActiveSubscriptionIdsReturnsEmptyWhenSessionNameIsUnknownAsync() + { + Mock session = CreateSession("operator"); + SetupRead( + session, + VariableIds.Server_ServerDiagnostics_SessionsDiagnosticsSummary_SessionDiagnosticsArray, + new DataValue(new Variant(new ArrayOf()), StatusCodes.Good)); + + var coordinator = new ClientFailoverCoordinator(); + ArrayOf ids = await coordinator + .DiscoverActiveSubscriptionIdsAsync( + session.Object, + new ClientRedundancyTransferOptions + { + ActiveSessionName = "missing" + }) + .ConfigureAwait(false); + + Assert.That(ids, Is.Empty); + session.Verify(s => s.TransferSubscriptionsAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [Test] + public async Task TransferActiveSubscriptionsReturnsEmptyWhenSubscriptionDiagnosticsAreUnavailableAsync() + { + var sessionId = new NodeId(42); + Mock session = CreateSession("operator"); + ArrayOf sessions = + [ + new ExtensionObject(new SessionDiagnosticsDataType + { + SessionId = sessionId, + SessionName = "active-client" + }) + ]; + SetupRead( + session, + VariableIds.Server_ServerDiagnostics_SessionsDiagnosticsSummary_SessionDiagnosticsArray, + new DataValue(new Variant(sessions), StatusCodes.Good)); + SetupRead( + session, + VariableIds.Server_ServerDiagnostics_SubscriptionDiagnosticsArray, + new DataValue(new Variant(new ArrayOf()), StatusCodes.Good)); + + var coordinator = new ClientFailoverCoordinator(); + ArrayOf results = await coordinator + .TransferActiveSubscriptionsAsync( + session.Object, + new ClientRedundancyTransferOptions + { + ActiveSessionName = "active-client" + }) + .ConfigureAwait(false); + + Assert.That(results, Is.Empty); + } + + [Test] + public void DiscoverActiveSubscriptionIdsRejectsNullArguments() + { + var coordinator = new ClientFailoverCoordinator(); + Mock session = CreateSession("operator"); + + Assert.That( + async () => await coordinator + .DiscoverActiveSubscriptionIdsAsync(null!, new ClientRedundancyTransferOptions()) + .ConfigureAwait(false), + Throws.ArgumentNullException); + Assert.That( + async () => await coordinator + .DiscoverActiveSubscriptionIdsAsync(session.Object, null!) + .ConfigureAwait(false), + Throws.ArgumentNullException); + } + + private static Mock CreateSession(string displayName) + { + var session = new Mock(); + session.SetupGet(s => s.Identity) + .Returns(new UserIdentity { DisplayName = displayName }); + return session; + } + + private static void SetupDiagnostics( + Mock session, + NodeId sessionId, + string sessionName, + uint[] subscriptionIds) + { + ArrayOf sessions = + [ + new ExtensionObject(new SessionDiagnosticsDataType + { + SessionId = sessionId, + SessionName = sessionName + }) + ]; + var subscriptions = new ArrayOf(); + foreach (uint subscriptionId in subscriptionIds) + { + subscriptions = subscriptions.AddItem( + new ExtensionObject(new SubscriptionDiagnosticsDataType + { + SessionId = sessionId, + SubscriptionId = subscriptionId + })); + } + + SetupRead( + session, + VariableIds.Server_ServerDiagnostics_SessionsDiagnosticsSummary_SessionDiagnosticsArray, + new DataValue(new Variant(sessions), StatusCodes.Good)); + SetupRead( + session, + VariableIds.Server_ServerDiagnostics_SubscriptionDiagnosticsArray, + new DataValue(new Variant(subscriptions), StatusCodes.Good)); + } + + private static void SetupRead( + Mock session, + NodeId nodeId, + DataValue value) + { + session.Setup(s => s.ReadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(r => r.Count == 1 && r[0].NodeId == nodeId), + It.IsAny())) + .ReturnsAsync(new ReadResponse + { + Results = [value], + DiagnosticInfos = [] + }); + } + } +} diff --git a/Tests/Opc.Ua.Client.Tests/Session/DefaultRedundantServerEndpointResolverTests.cs b/Tests/Opc.Ua.Client.Tests/Session/DefaultRedundantServerEndpointResolverTests.cs new file mode 100644 index 0000000000..af9df4c640 --- /dev/null +++ b/Tests/Opc.Ua.Client.Tests/Session/DefaultRedundantServerEndpointResolverTests.cs @@ -0,0 +1,473 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Client.Tests.ManagedSession +{ + /// + /// Tests for . + /// + [TestFixture] + [Category("Client")] + [Category("ServerRedundancy")] + public sealed class DefaultRedundantServerEndpointResolverTests + { + [Test] + public void ResolveAsyncRejectsInvalidArguments() + { + var resolver = new DefaultRedundantServerEndpointResolver(); + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", "opc.tcp://server:4840"); + + Assert.That( + async () => await resolver.ResolveAsync(string.Empty, endpoint).ConfigureAwait(false), + Throws.ArgumentException); + Assert.That( + async () => await resolver.ResolveAsync("urn:server", null!).ConfigureAwait(false), + Throws.ArgumentNullException); + } + + [Test] + public async Task ResolveAsyncReturnsNullWhenCurrentEndpointHasNoDiscoveryUrlAsync() + { + var resolver = new DefaultRedundantServerEndpointResolver(); + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", string.Empty); + + ConfiguredEndpoint? result = await resolver + .ResolveAsync("urn:peer", endpoint) + .ConfigureAwait(false); + + Assert.That(result, Is.Null); + } + + [Test] + public async Task ResolveAsyncSelectsEndpointWithMatchingSecurityAsync() + { + ConfiguredEndpoint currentEndpoint = CreateEndpoint( + "urn:current", + "opc.tcp://current:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + ApplicationDescription peer = CreateApplication("urn:peer", "opc.tcp://peer:4840"); + EndpointDescription downgrade = CreateEndpointDescription( + "urn:peer", + "opc.tcp://peer:4840", + MessageSecurityMode.None, + SecurityPolicies.None); + EndpointDescription matching = CreateEndpointDescription( + "urn:peer", + "opc.tcp://peer:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + RecordingDiscovery discovery = CreateDiscovery( + [peer], + [downgrade, matching]); + var resolver = new DefaultRedundantServerEndpointResolver( + NUnitTelemetryContext.Create(), + discovery); + + ConfiguredEndpoint? result = await resolver + .ResolveAsync("urn:peer", currentEndpoint) + .ConfigureAwait(false); + + Assert.That(result, Is.Not.Null); + Assert.That(result!.Description.SecurityMode, Is.EqualTo(MessageSecurityMode.Sign)); + Assert.That(result.Description.SecurityPolicyUri, Is.EqualTo(SecurityPolicies.Basic256Sha256)); + Assert.That(result.Description.Server.ApplicationUri, Is.EqualTo("urn:peer")); + } + + [Test] + public async Task ResolveAsyncRejectsSecurityDowngradeEndpointAsync() + { + ConfiguredEndpoint currentEndpoint = CreateEndpoint( + "urn:current", + "opc.tcp://current:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + ApplicationDescription peer = CreateApplication("urn:peer", "opc.tcp://peer:4840"); + EndpointDescription downgrade = CreateEndpointDescription( + "urn:peer", + "opc.tcp://peer:4840", + MessageSecurityMode.None, + SecurityPolicies.None); + RecordingDiscovery discovery = CreateDiscovery([peer], [downgrade]); + var resolver = new DefaultRedundantServerEndpointResolver( + NUnitTelemetryContext.Create(), + discovery); + + ConfiguredEndpoint? result = await resolver + .ResolveAsync("urn:peer", currentEndpoint) + .ConfigureAwait(false); + + Assert.That(result, Is.Null); + } + + [Test] + public async Task ResolveAsyncReturnsNullWhenFindServersDoesNotReturnPeerAsync() + { + ConfiguredEndpoint currentEndpoint = CreateEndpoint("urn:current", "opc.tcp://current:4840"); + RecordingDiscovery discovery = CreateDiscovery( + [CreateApplication("urn:other", "opc.tcp://other:4840")], + []); + var resolver = new DefaultRedundantServerEndpointResolver( + NUnitTelemetryContext.Create(), + discovery); + + ConfiguredEndpoint? result = await resolver + .ResolveAsync("urn:peer", currentEndpoint) + .ConfigureAwait(false); + + Assert.That(result, Is.Null); + Assert.That(discovery.GetEndpointsCallCount, Is.Zero); + } + + [Test] + public async Task ResolveAsyncTriesNextDiscoveryUrlWhenFirstHasNoMatchingEndpointAsync() + { + ConfiguredEndpoint currentEndpoint = CreateEndpoint( + "urn:current", + "opc.tcp://current:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + currentEndpoint.Description.Server.DiscoveryUrls = + [ + "opc.tcp://discovery-a:4840", + "opc.tcp://discovery-b:4840" + ]; + ApplicationDescription peer = CreateApplication("urn:peer", "opc.tcp://peer:4840"); + EndpointDescription matching = CreateEndpointDescription( + "urn:peer", + "opc.tcp://peer:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + var discovery = new RecordingDiscovery( + [peer], + [[], new ArrayOf(new[] { matching })]); + var resolver = new DefaultRedundantServerEndpointResolver( + NUnitTelemetryContext.Create(), + discovery); + + ConfiguredEndpoint? result = await resolver + .ResolveAsync("urn:peer", currentEndpoint) + .ConfigureAwait(false); + + Assert.That(result, Is.Not.Null); + Assert.That(discovery.GetEndpointsCallCount, Is.EqualTo(2)); + } + + [Test] + public void PrivateSelectionHelpersMatchSecurityAndDiscoveryFallbacks() + { + EndpointDescription current = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + EndpointDescription same = CreateEndpointDescription( + "urn:server", + "opc.tcp://backup:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + EndpointDescription different = CreateEndpointDescription( + "urn:server", + "https://backup:443", + MessageSecurityMode.None, + SecurityPolicies.None); + + Assert.That(InvokeBool("IsSameScheme", same, current), Is.True); + Assert.That(InvokeBool("IsSameSecurity", same, current), Is.True); + Assert.That(InvokeBool("IsSameScheme", different, current), Is.False); + Assert.That(InvokeBool("IsSameSecurity", different, current), Is.False); + + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", "opc.tcp://server:4840"); + endpoint.Description.Server.DiscoveryUrls = ["opc.tcp://discovery:4840"]; + Assert.That(InvokeDiscoveryUrls(endpoint), Is.EqualTo(s_discoveryUrls)); + + endpoint.Description.Server.DiscoveryUrls = []; + Assert.That(InvokeDiscoveryUrls(endpoint), Is.EqualTo(s_endpointUrls)); + } + + [Test] + public void IsSameSchemeMissingUrisReturnsFalse() + { + EndpointDescription current = CreateEndpointDescription( + "urn:server", + string.Empty, + MessageSecurityMode.None, + SecurityPolicies.None); + EndpointDescription candidate = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.None, + SecurityPolicies.None); + + Assert.That(InvokeBool("IsSameScheme", candidate, current), Is.False); + Assert.That(InvokeBool("IsSameScheme", current, candidate), Is.False); + } + + [Test] + public void IsSameSecurityMatchesSignAndEncryptEndpoints() + { + EndpointDescription endpoint1 = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.SignAndEncrypt, + SecurityPolicies.Basic256Sha256); + EndpointDescription endpoint2 = CreateEndpointDescription( + "urn:server", + "opc.tcp://backup:4840", + MessageSecurityMode.SignAndEncrypt, + SecurityPolicies.Basic256Sha256); + + Assert.That(InvokeBool("IsSameSecurity", endpoint1, endpoint2), Is.True); + } + + [Test] + public void IsSameSecurityDifferentiatesSecurityModes() + { + EndpointDescription signMode = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.Sign, + SecurityPolicies.Basic256Sha256); + EndpointDescription noneMode = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.None, + SecurityPolicies.None); + + Assert.That(InvokeBool("IsSameSecurity", signMode, noneMode), Is.False); + } + + [Test] + public void IsSameSecurityDifferentiatesSecurityPolicies() + { + EndpointDescription basic256 = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.SignAndEncrypt, + SecurityPolicies.Basic256Sha256); + EndpointDescription aes128 = CreateEndpointDescription( + "urn:server", + "opc.tcp://server:4840", + MessageSecurityMode.SignAndEncrypt, + SecurityPolicies.Aes128_Sha256_RsaOaep); + + Assert.That(InvokeBool("IsSameSecurity", basic256, aes128), Is.False); + } + + [Test] + public void GetDiscoveryUrlsHandlesEmptyEndpointUrl() + { + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", string.Empty); + endpoint.Description.Server = new ApplicationDescription { ApplicationUri = "urn:server" }; + + string[] urls = InvokeDiscoveryUrls(endpoint); + + Assert.That(urls, Has.Length.EqualTo(0)); + } + + [Test] + public void GetDiscoveryUrlsReturnsEndpointUrlWhenNoServerDiscoveryUrls() + { + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", "opc.tcp://server:4840"); + endpoint.Description.Server.DiscoveryUrls = []; + + string[] urls = InvokeDiscoveryUrls(endpoint); + + Assert.That(urls, Has.Length.EqualTo(1)); + Assert.That(urls[0], Is.EqualTo("opc.tcp://server:4840")); + } + + [Test] + public void GetDiscoveryUrlsReturnsServerDiscoveryUrlsWhenAvailable() + { + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", "opc.tcp://server:4840"); + endpoint.Description.Server.DiscoveryUrls = ["opc.tcp://discovery:4840", "https://discovery:443"]; + + string[] urls = InvokeDiscoveryUrls(endpoint); + + Assert.That(urls, Has.Length.EqualTo(2)); + Assert.That(urls[0], Is.EqualTo("opc.tcp://discovery:4840")); + Assert.That(urls[1], Is.EqualTo("https://discovery:443")); + } + + [Test] + public void GetDiscoveryUrlsHandlesNullServer() + { + ConfiguredEndpoint endpoint = CreateEndpoint("urn:server", "opc.tcp://server:4840"); + endpoint.Description.Server = null!; + + string[] urls = InvokeDiscoveryUrls(endpoint); + + Assert.That(urls, Has.Length.EqualTo(1)); + Assert.That(urls[0], Is.EqualTo("opc.tcp://server:4840")); + } + + private static bool InvokeBool( + string methodName, + EndpointDescription endpoint, + EndpointDescription currentEndpoint) + { + MethodInfo method = typeof(DefaultRedundantServerEndpointResolver).GetMethod( + methodName, + BindingFlags.NonPublic | BindingFlags.Static)!; + return (bool)method.Invoke(null, [endpoint, currentEndpoint])!; + } + + private static string[] InvokeDiscoveryUrls(ConfiguredEndpoint endpoint) + { + MethodInfo method = typeof(DefaultRedundantServerEndpointResolver).GetMethod( + "GetDiscoveryUrls", + BindingFlags.NonPublic | BindingFlags.Static)!; + var urls = (IEnumerable)method.Invoke(null, [endpoint])!; + var result = new List(); + foreach (string url in urls) + { + result.Add(url); + } + + return result.ToArray(); + } + + private static ConfiguredEndpoint CreateEndpoint(string serverUri, string endpointUrl) + { + return CreateEndpoint( + serverUri, + endpointUrl, + MessageSecurityMode.None, + SecurityPolicies.None); + } + + private static ConfiguredEndpoint CreateEndpoint( + string serverUri, + string endpointUrl, + MessageSecurityMode securityMode, + string securityPolicyUri) + { + return new ConfiguredEndpoint( + null, + CreateEndpointDescription( + serverUri, + endpointUrl, + securityMode, + securityPolicyUri), + configuration: null); + } + + private static EndpointDescription CreateEndpointDescription( + string serverUri, + string endpointUrl, + MessageSecurityMode securityMode, + string securityPolicyUri) + { + return new EndpointDescription + { + EndpointUrl = endpointUrl, + SecurityMode = securityMode, + SecurityPolicyUri = securityPolicyUri, + Server = new ApplicationDescription + { + ApplicationUri = serverUri, + DiscoveryUrls = string.IsNullOrEmpty(endpointUrl) + ? [] + : [endpointUrl] + } + }; + } + + private static ApplicationDescription CreateApplication( + string serverUri, + string discoveryUrl) + { + return new ApplicationDescription + { + ApplicationUri = serverUri, + DiscoveryUrls = [discoveryUrl] + }; + } + + private static RecordingDiscovery CreateDiscovery( + ApplicationDescription[] applications, + EndpointDescription[] endpoints) + { + return new RecordingDiscovery( + new ArrayOf(applications), + [new ArrayOf(endpoints)]); + } + + private sealed class RecordingDiscovery : IRedundantServerDiscovery + { + public RecordingDiscovery( + ArrayOf applications, + ArrayOf[] endpointResponses) + { + m_applications = applications; + m_endpointResponses = endpointResponses; + } + + public int GetEndpointsCallCount { get; private set; } + + public ValueTask> FindServersAsync( + Uri discoveryUri, + EndpointConfiguration configuration, + string serverUri, + ITelemetryContext telemetry, + CancellationToken ct) + { + return new ValueTask>(m_applications); + } + + public ValueTask> GetEndpointsAsync( + Uri discoveryUri, + EndpointConfiguration configuration, + ITelemetryContext telemetry, + CancellationToken ct) + { + int index = Math.Min(GetEndpointsCallCount, m_endpointResponses.Length - 1); + GetEndpointsCallCount++; + return new ValueTask>(m_endpointResponses[index]); + } + + private readonly ArrayOf m_applications; + private readonly ArrayOf[] m_endpointResponses; + } + + private static readonly string[] s_discoveryUrls = ["opc.tcp://discovery:4840"]; + private static readonly string[] s_endpointUrls = ["opc.tcp://server:4840"]; + } +} diff --git a/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionComplianceTests.cs b/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionComplianceTests.cs index 9a54d9d408..b3f609fba6 100644 --- a/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionComplianceTests.cs +++ b/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionComplianceTests.cs @@ -732,6 +732,8 @@ private static ManagedSessionClass typeof(bool), typeof(bool), typeof(bool), + typeof(bool), + typeof(NetworkRedundancyOptions), typeof(IClientChannelManager) ], null); @@ -756,6 +758,8 @@ private static ManagedSessionClass false, false, false, + false, + null, null ]); diff --git a/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionTests.cs b/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionTests.cs index 9fad92fd0e..d32e1b6b88 100644 --- a/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionTests.cs +++ b/Tests/Opc.Ua.Client.Tests/Session/ManagedSessionTests.cs @@ -400,6 +400,8 @@ private static Client.ManagedSession CreateManagedSessionWithInner( typeof(bool), typeof(bool), typeof(bool), + typeof(bool), + typeof(NetworkRedundancyOptions), typeof(IClientChannelManager) ], null); @@ -423,6 +425,8 @@ private static Client.ManagedSession CreateManagedSessionWithInner( false, false, false, + false, + null, null ]); diff --git a/Tests/Opc.Ua.Client.Tests/Session/NetworkRedundancyEndpointSelectorTests.cs b/Tests/Opc.Ua.Client.Tests/Session/NetworkRedundancyEndpointSelectorTests.cs new file mode 100644 index 0000000000..13c7a1e23c --- /dev/null +++ b/Tests/Opc.Ua.Client.Tests/Session/NetworkRedundancyEndpointSelectorTests.cs @@ -0,0 +1,145 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using NUnit.Framework; + +namespace Opc.Ua.Client.Tests.ManagedSession +{ + /// + /// Tests for . + /// + [TestFixture] + [Category("Client")] + [Category("NetworkRedundancy")] + public sealed class NetworkRedundancyEndpointSelectorTests + { + [Test] + public void SelectNextReturnsAlternateEndpointForSameLogicalServer() + { + ConfiguredEndpoint primary = CreateEndpoint("urn:server", "opc.tcp://path-a:4840"); + ConfiguredEndpoint alternate = CreateEndpoint("urn:server", "opc.tcp://path-b:4840"); + var selector = new NetworkRedundancyEndpointSelector( + primary, + [alternate]); + + ConfiguredEndpoint? next = selector.SelectNext(primary); + + Assert.That(next, Is.SameAs(alternate)); + } + + [Test] + public void SelectNextWrapsAndKeepsSameLogicalSession() + { + ConfiguredEndpoint primary = CreateEndpoint("urn:server", "opc.tcp://path-a:4840"); + ConfiguredEndpoint alternate = CreateEndpoint("urn:server", "opc.tcp://path-b:4840"); + var selector = new NetworkRedundancyEndpointSelector( + primary, + [alternate]); + + ConfiguredEndpoint? next = selector.SelectNext(alternate); + + Assert.That(next, Is.SameAs(primary)); + Assert.That(next!.Description.Server.ApplicationUri, Is.EqualTo("urn:server")); + } + + [Test] + public void SelectNextIgnoresDifferentLogicalServer() + { + ConfiguredEndpoint primary = CreateEndpoint("urn:server", "opc.tcp://path-a:4840"); + ConfiguredEndpoint otherServer = CreateEndpoint("urn:other", "opc.tcp://path-b:4840"); + var selector = new NetworkRedundancyEndpointSelector( + primary, + [otherServer]); + + Assert.That(selector.HasAlternates, Is.False); + Assert.That(selector.SelectNext(primary), Is.Null); + } + + [Test] + public void SelectNextReturnsPrimaryWhenCurrentEndpointIsUnknown() + { + ConfiguredEndpoint primary = CreateEndpoint("urn:server", "opc.tcp://path-a:4840"); + ConfiguredEndpoint alternate = CreateEndpoint("urn:server", "opc.tcp://path-b:4840"); + ConfiguredEndpoint unknown = CreateEndpoint("urn:server", "opc.tcp://path-c:4840"); + var selector = new NetworkRedundancyEndpointSelector( + primary, + [alternate]); + + ConfiguredEndpoint? next = selector.SelectNext(unknown); + + Assert.That(next, Is.SameAs(primary)); + } + + [Test] + public void SelectNextDoesNotAddDuplicateAlternateEndpoint() + { + ConfiguredEndpoint primary = CreateEndpoint("urn:server", "opc.tcp://path-a:4840"); + ConfiguredEndpoint alternate = CreateEndpoint("urn:server", "opc.tcp://path-b:4840"); + ConfiguredEndpoint duplicate = CreateEndpoint("urn:server", "opc.tcp://PATH-B:4840"); + var selector = new NetworkRedundancyEndpointSelector( + primary, + [alternate, duplicate]); + + ConfiguredEndpoint? next = selector.SelectNext(alternate); + + Assert.That(next, Is.SameAs(primary)); + } + + [Test] + public void SelectNextAllowsMissingApplicationUriForSameLogicalServer() + { + ConfiguredEndpoint primary = CreateEndpoint(string.Empty, "opc.tcp://path-a:4840"); + ConfiguredEndpoint alternate = CreateEndpoint("urn:server", "opc.tcp://path-b:4840"); + var selector = new NetworkRedundancyEndpointSelector( + primary, + [alternate]); + + Assert.That(selector.HasAlternates, Is.True); + Assert.That(selector.SelectNext(primary), Is.SameAs(alternate)); + } + + private static ConfiguredEndpoint CreateEndpoint(string serverUri, string endpointUrl) + { + var description = new EndpointDescription + { + EndpointUrl = endpointUrl, + SecurityMode = MessageSecurityMode.None, + SecurityPolicyUri = SecurityPolicies.None, + Server = new ApplicationDescription + { + ApplicationUri = serverUri + } + }; + + return new ConfiguredEndpoint(null, description, configuration: null); + } + } +} diff --git a/Tests/Opc.Ua.Client.Tests/Session/ServerRedundancyHandlerTests.cs b/Tests/Opc.Ua.Client.Tests/Session/ServerRedundancyHandlerTests.cs index ee0bb07d4b..c5e50b0556 100644 --- a/Tests/Opc.Ua.Client.Tests/Session/ServerRedundancyHandlerTests.cs +++ b/Tests/Opc.Ua.Client.Tests/Session/ServerRedundancyHandlerTests.cs @@ -30,6 +30,7 @@ #nullable enable using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Moq; @@ -45,132 +46,245 @@ namespace Opc.Ua.Client.Tests.ManagedSession [Category("ServerRedundancy")] public sealed class ServerRedundancyHandlerTests { - private DefaultServerRedundancyHandler m_handler; + private static readonly DateTime s_now = new(2026, 6, 27, 4, 0, 0, DateTimeKind.Utc); + private Mock m_resolver = null!; + private DefaultServerRedundancyHandler m_handler = null!; [SetUp] public void SetUp() { - m_handler = new DefaultServerRedundancyHandler(); + m_resolver = new Mock(MockBehavior.Strict); + m_handler = new DefaultServerRedundancyHandler(m_resolver.Object, new FixedTimeProvider(s_now)); + } + + [TestCase(0, RedundancySupport.None)] + [TestCase(1, RedundancySupport.Cold)] + [TestCase(2, RedundancySupport.Warm)] + [TestCase(3, RedundancySupport.Hot)] + [TestCase(4, RedundancySupport.Transparent)] + [TestCase(5, RedundancySupport.HotAndMirrored)] + public async Task FetchRedundancyInfoMapsRedundancySupportAsync( + int value, + RedundancySupport expected) + { + Mock mockSession = CreateMockSession( + redundancySupport: value, + serviceLevel: ServiceLevels.HealthyMinimum); + + ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( + mockSession.Object).ConfigureAwait(false); + + Assert.That(info.Mode, Is.EqualTo(expected)); } [Test] - public void SelectFailoverTargetReturnsNullForNoneMode() + public void ShouldFailoverStaysWhileCurrentServerIsHealthy() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.None, - ServiceLevel = 200, - RedundantServers = [] + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.HealthyMinimum, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Healthy, + RedundantServers = + [ + CreateServerInfo("urn:backup", ServiceLevels.Maximum, ServerState.Running) + ] }; - ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( + ServerFailoverDecision decision = m_handler.ShouldFailover( + info, CreateCurrentEndpoint("urn:current")); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Null); + Assert.That(decision.IsFailoverWarranted, Is.False); + Assert.That(target, Is.Null); } [Test] - public void SelectFailoverTargetReturnsNullForTransparentMode() + public void ShouldFailoverSwitchesFromDegradedToHealthyPeer() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Transparent, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.DegradedMaximum, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Degraded, RedundantServers = [ - CreateServerInfo("urn:backup", 200, ServerState.Running) + CreateServerInfo("urn:degraded", 150, ServerState.Running), + CreateServerInfo("urn:healthy", 230, ServerState.Running) ] }; - ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( + ServerFailoverDecision decision = m_handler.ShouldFailover( + info, CreateCurrentEndpoint("urn:current")); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Null); + Assert.That(decision.IsFailoverWarranted, Is.True); + Assert.That(target, Is.Not.Null); + Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy")); } [Test] - public void SelectFailoverTargetSelectsHighestServiceLevel() + public void ShouldFailoverDoesNotSwitchFromDegradedWithoutHealthyPeer() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Hot, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = 100, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Degraded, RedundantServers = [ - CreateServerInfo("urn:server-low", 100, ServerState.Running), - CreateServerInfo("urn:server-high", 250, ServerState.Running), - CreateServerInfo("urn:server-mid", 180, ServerState.Running) + CreateServerInfo("urn:degraded", 150, ServerState.Running) ] }; - ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( + ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Not.Null); - Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:server-high")); + Assert.That(decision.IsFailoverWarranted, Is.False); } [Test] - public void SelectFailoverTargetSkipsCurrentEndpoint() + public void ShouldFailoverSwitchesFromMaintenanceToOperationalPeer() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Hot, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.Maintenance, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, + EstimatedReturnTime = s_now.AddMinutes(10), RedundantServers = [ - CreateServerInfo("urn:current", 255, ServerState.Running), - CreateServerInfo("urn:backup", 100, ServerState.Running) + CreateServerInfo("urn:healthy", 230, ServerState.Running) ] }; - ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( + ServerFailoverDecision decision = m_handler.ShouldFailover( + info, CreateCurrentEndpoint("urn:current")); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Not.Null); + Assert.That(decision.IsFailoverWarranted, Is.True); + Assert.That(target, Is.Not.Null); + Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy")); + } + + [Test] + public void ShouldFailoverHonorsMaintenanceEstimatedReturnTime() + { + DateTime estimatedReturnTime = s_now.AddMinutes(10); + var info = new ServerRedundancyInfo + { + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.Maintenance, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, + EstimatedReturnTime = estimatedReturnTime, + RedundantServers = + [ + CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown) + ] + }; + + ServerFailoverDecision decision = m_handler.ShouldFailover( + info, CreateCurrentEndpoint("urn:current")); + + Assert.That(decision.IsFailoverWarranted, Is.False); + Assert.That(decision.RetryAfter, Is.EqualTo(estimatedReturnTime)); + } + + [Test] + public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeLapsed() + { + var info = new ServerRedundancyInfo + { + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.Maintenance, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, + EstimatedReturnTime = s_now.AddMinutes(-1), + RedundantServers = + [ + CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown) + ] + }; + + ServerFailoverDecision decision = m_handler.ShouldFailover( + info, CreateCurrentEndpoint("urn:current")); + + Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:backup")); + decision.RetryAfter, + Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff))); } [Test] - public void SelectFailoverTargetSkipsNonRunningServers() + public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeIsAbsent() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Hot, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.Maintenance, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, + EstimatedReturnTime = DateTime.MinValue, RedundantServers = [ - CreateServerInfo("urn:suspended", 255, ServerState.Suspended), - CreateServerInfo("urn:shutdown", 240, ServerState.Shutdown), - CreateServerInfo("urn:running", 100, ServerState.Running) + CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown) ] }; - ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( + ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Not.Null); + Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:running")); + decision.RetryAfter, + Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff))); } [Test] - public void SelectFailoverTargetReturnsNullWhenNoViableServers() + public void ShouldFailoverReturnsNoFailoverWhenAllPeersAreDown() + { + var info = new ServerRedundancyInfo + { + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.NoData, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.NoData, + RedundantServers = + [ + CreateServerInfo("urn:shutdown", ServiceLevels.Maximum, ServerState.Shutdown), + CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended) + ] + }; + + ServerFailoverDecision decision = m_handler.ShouldFailover( + info, CreateCurrentEndpoint("urn:current")); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( + info, CreateCurrentEndpoint("urn:current")); + + Assert.That(decision.IsFailoverWarranted, Is.False); + Assert.That(target, Is.Null); + } + + [Test] + public void SelectFailoverTargetReturnsNullForTransparentMode() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Hot, - ServiceLevel = 200, + Mode = RedundancySupport.Transparent, + ServiceLevel = ServiceLevels.NoData, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ - CreateServerInfo("urn:current", 200, ServerState.Running), - CreateServerInfo("urn:down", 100, ServerState.Shutdown), - CreateServerInfo("urn:failed", 50, ServerState.Failed) + CreateServerInfo("urn:backup", 230, ServerState.Running) ] }; @@ -181,37 +295,39 @@ public void SelectFailoverTargetReturnsNullWhenNoViableServers() } [Test] - public void SelectFailoverTargetWorksForColdMode() + public void SelectFailoverTargetReturnsNullForNoneMode() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Cold, - ServiceLevel = 200, + Mode = RedundancySupport.None, + ServiceLevel = ServiceLevels.NoData, + ServiceLevelAccessible = false, + ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ - CreateServerInfo("urn:backup", 150, ServerState.Running) + CreateServerInfo("urn:backup", 230, ServerState.Running) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Not.Null); - Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:backup")); + Assert.That(result, Is.Null); } [Test] - public void SelectFailoverTargetWorksForWarmMode() + public void SelectFailoverTargetSkipsCurrentEndpoint() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Warm, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.NoData, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ - CreateServerInfo("urn:backup", 150, ServerState.Running) + CreateServerInfo("urn:current", ServiceLevels.Maximum, ServerState.Running), + CreateServerInfo("urn:backup", 230, ServerState.Running) ] }; @@ -219,21 +335,23 @@ public void SelectFailoverTargetWorksForWarmMode() info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Not.Null); - Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:backup")); + Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:backup")); } [Test] - public void SelectFailoverTargetWorksForHotMode() + public void SelectFailoverTargetSkipsNonRunningServers() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.Hot, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.NoData, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ - CreateServerInfo("urn:backup", 150, ServerState.Running) + CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended), + CreateServerInfo("urn:shutdown", 240, ServerState.Shutdown), + CreateServerInfo("urn:running", 230, ServerState.Running) ] }; @@ -241,99 +359,212 @@ public void SelectFailoverTargetWorksForHotMode() info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Not.Null); - Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:backup")); + Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:running")); } [Test] - public void SelectFailoverTargetWorksForHotAndMirroredMode() + public void SelectFailoverTargetReturnsNullWhenNoViableServers() { var info = new ServerRedundancyInfo { - Mode = RedundancyMode.HotAndMirrored, - ServiceLevel = 200, + Mode = RedundancySupport.Hot, + ServiceLevel = ServiceLevels.NoData, + ServiceLevelAccessible = true, + ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ - CreateServerInfo("urn:backup", 150, ServerState.Running) + CreateServerInfo("urn:current", 230, ServerState.Running), + CreateServerInfo("urn:down", 230, ServerState.Shutdown) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); - Assert.That(result, Is.Not.Null); + Assert.That(result, Is.Null); + } + + [Test] + public async Task FetchRedundancyInfoReadsRedundantServerArrayAsync() + { + var serverData = new RedundantServerDataType + { + ServerId = "urn:server1", + ServiceLevel = 230, + ServerState = ServerState.Running + }; + ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840"); + m_resolver.Setup(r => r.ResolveAsync( + "urn:server1", + It.IsAny(), + It.IsAny())) + .ReturnsAsync(resolvedEndpoint); + + Mock mockSession = CreateMockSession( + redundancySupport: (int)RedundancySupport.Hot, + serviceLevel: 100, + redundantServers: [serverData]); + + ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( + mockSession.Object).ConfigureAwait(false); + + Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot)); + Assert.That(info.ServiceLevelSubrange, Is.EqualTo(ServiceLevelSubrange.Degraded)); + Assert.That(info.RedundantServers, Has.Count.EqualTo(1)); + Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server1")); + Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint)); + } + + [Test] + public async Task FetchRedundancyInfoResolvesServerUriArrayToEndpointsAsync() + { + ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840"); + m_resolver.Setup(r => r.ResolveAsync( + "urn:server-uri", + It.IsAny(), + It.IsAny())) + .ReturnsAsync(resolvedEndpoint); + + Mock mockSession = CreateMockSession( + redundancySupport: (int)RedundancySupport.Hot, + serviceLevel: ServiceLevels.NoData, + serverUris: ["urn:server-uri"]); + + ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( + mockSession.Object).ConfigureAwait(false); + + Assert.That(info.RedundantServers, Has.Count.EqualTo(1)); + Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server-uri")); + Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint)); Assert.That( - result!.Description.Server.ApplicationUri, - Is.EqualTo("urn:backup")); + info.RedundantServers[0].Endpoint!.Description.EndpointUrl, + Is.EqualTo("opc.tcp://server-uri:4840")); + VerifyBatchedRedundancyRead(mockSession); } [Test] - public async Task FetchRedundancyInfoReturnsNoneWhenNotSupportedAsync() + public async Task FetchRedundancyInfoExcludesUnresolvedServerUriFromSelectionAsync() { + m_resolver.Setup(r => r.ResolveAsync( + "urn:unresolved", + It.IsAny(), + It.IsAny())) + .ReturnsAsync((ConfiguredEndpoint?)null); Mock mockSession = CreateMockSession( - redundancySupport: (int)RedundancySupport.None, - serviceLevel: 200); + redundancySupport: (int)RedundancySupport.Hot, + serviceLevel: ServiceLevels.NoData, + serverUris: ["urn:unresolved"]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( + info, CreateCurrentEndpoint("urn:current")); - Assert.That(info.Mode, Is.EqualTo(RedundancyMode.None)); - Assert.That(info.RedundantServers, Is.Empty); + Assert.That(info.RedundantServers, Has.Count.EqualTo(1)); + Assert.That(info.RedundantServers[0].Endpoint, Is.Null); + Assert.That(target, Is.Null); } [Test] - public async Task FetchRedundancyInfoReadsServiceLevelAsync() + public async Task FetchRedundancyInfoSelectsServerUriOnlyPeerWithUnknownServiceLevelAsync() { + ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840"); + m_resolver.Setup(r => r.ResolveAsync( + "urn:server-uri", + It.IsAny(), + It.IsAny())) + .ReturnsAsync(resolvedEndpoint); Mock mockSession = CreateMockSession( - redundancySupport: (int)RedundancySupport.None, - serviceLevel: 175); + redundancySupport: (int)RedundancySupport.Hot, + serviceLevel: ServiceLevels.NoData, + serverUris: ["urn:server-uri"]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( + info, CreateCurrentEndpoint("urn:current")); - Assert.That(info.ServiceLevel, Is.EqualTo(175)); + Assert.That(info.RedundantServers[0].ServiceLevel, Is.EqualTo(ServiceLevels.NoData)); + Assert.That(target, Is.SameAs(resolvedEndpoint)); } [Test] - public async Task FetchRedundancyInfoReadsRedundantServerArrayAsync() + public async Task FetchRedundancyInfoFallsBackToUnknownServerUriOnlyPeerAsync() { - var serverData1 = new RedundantServerDataType + var serverData = new RedundantServerDataType { - ServerId = "urn:server1", - ServiceLevel = 200, + ServerId = "urn:nodata", + ServiceLevel = ServiceLevels.NoData, ServerState = ServerState.Running }; - var serverData2 = new RedundantServerDataType + ConfiguredEndpoint noDataEndpoint = CreateEndpoint("urn:nodata", "opc.tcp://nodata:4840"); + ConfiguredEndpoint unknownEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840"); + m_resolver.Setup(r => r.ResolveAsync( + "urn:nodata", + It.IsAny(), + It.IsAny())) + .ReturnsAsync(noDataEndpoint); + m_resolver.Setup(r => r.ResolveAsync( + "urn:server-uri", + It.IsAny(), + It.IsAny())) + .ReturnsAsync(unknownEndpoint); + Mock mockSession = CreateMockSession( + redundancySupport: (int)RedundancySupport.Hot, + serviceLevel: ServiceLevels.DegradedMaximum, + redundantServers: [serverData], + serverUris: ["urn:nodata", "urn:server-uri"]); + + ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( + mockSession.Object).ConfigureAwait(false); + ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( + info, CreateCurrentEndpoint("urn:current")); + + Assert.That(info.RedundantServers, Has.Count.EqualTo(2)); + Assert.That(target, Is.SameAs(unknownEndpoint)); + } + + [Test] + public async Task FetchRedundancyInfoCachesResolvedEndpointsAsync() + { + var serverData = new RedundantServerDataType { - ServerId = "urn:server2", - ServiceLevel = 150, - ServerState = ServerState.Suspended + ServerId = "urn:server1", + ServiceLevel = 230, + ServerState = ServerState.Running }; - + ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840"); + m_resolver.Setup(r => r.ResolveAsync( + "urn:server1", + It.IsAny(), + It.IsAny())) + .ReturnsAsync(resolvedEndpoint); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, - serviceLevel: 200, - redundantServers: [serverData1, serverData2]); + serviceLevel: 100, + redundantServers: [serverData]); + + await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false); + await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false); + + m_resolver.Verify(r => r.ResolveAsync( + "urn:server1", + It.IsAny(), + It.IsAny()), Times.Once); + } + + [Test] + public async Task FetchRedundancyInfoReadsTransparentCurrentServerIdAsync() + { + Mock mockSession = CreateMockSession( + redundancySupport: (int)RedundancySupport.Transparent, + serviceLevel: ServiceLevels.HealthyMinimum, + currentServerId: "server-a"); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); - Assert.That(info.Mode, Is.EqualTo(RedundancyMode.Hot)); - Assert.That(info.RedundantServers, Has.Count.EqualTo(2)); - Assert.That( - info.RedundantServers[0].ServerUri, - Is.EqualTo("urn:server1")); - Assert.That(info.RedundantServers[0].ServiceLevel, Is.EqualTo(200)); - Assert.That( - info.RedundantServers[0].ServerState, - Is.EqualTo(ServerState.Running)); - Assert.That( - info.RedundantServers[1].ServerUri, - Is.EqualTo("urn:server2")); - Assert.That( - info.RedundantServers[1].ServerState, - Is.EqualTo(ServerState.Suspended)); + Assert.That(info.CurrentServerId, Is.EqualTo("server-a")); } [Test] @@ -344,96 +575,165 @@ public async Task FetchRedundancyInfoHandlesReadErrorsAsync() ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); - Assert.That(info.Mode, Is.EqualTo(RedundancyMode.None)); + Assert.That(info.Mode, Is.EqualTo(RedundancySupport.None)); Assert.That(info.ServiceLevel, Is.Zero); + Assert.That(info.ServiceLevelAccessible, Is.False); Assert.That(info.RedundantServers, Is.Empty); } + [Test] + public async Task FetchRedundancyInfoHandlesMalformedOptionalNodesAsync() + { + Mock mockSession = CreateMockSessionWithThrowingOptionalReads(); + + ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( + mockSession.Object).ConfigureAwait(false); + + Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot)); + Assert.That(info.RedundantServers, Is.Empty); + } + + [Test] + public void NullArgumentsThrow() + { + var info = new ServerRedundancyInfo(); + ConfiguredEndpoint endpoint = CreateCurrentEndpoint("urn:current"); + + Assert.That( + async () => await m_handler.FetchRedundancyInfoAsync(null!).ConfigureAwait(false), + Throws.ArgumentNullException); + Assert.That(() => m_handler.ShouldFailover(null!, endpoint), Throws.ArgumentNullException); + Assert.That(() => m_handler.ShouldFailover(info, null!), Throws.ArgumentNullException); + Assert.That(() => m_handler.SelectFailoverTarget(null!, endpoint), Throws.ArgumentNullException); + Assert.That(() => m_handler.SelectFailoverTarget(info, null!), Throws.ArgumentNullException); + } + private static RedundantServer CreateServerInfo( - string uri, byte serviceLevel, ServerState state) + string uri, + byte serviceLevel, + ServerState state) { return new RedundantServer { ServerUri = uri, ServiceLevel = serviceLevel, - ServerState = state + ServerState = state, + Endpoint = CreateEndpoint(uri, $"opc.tcp://{uri[4..]}:4840") }; } private static ConfiguredEndpoint CreateCurrentEndpoint(string applicationUri) + { + return CreateEndpoint(applicationUri, "opc.tcp://current:4840"); + } + + private static ConfiguredEndpoint CreateEndpoint(string applicationUri, string endpointUrl) { var description = new EndpointDescription { - EndpointUrl = applicationUri, + EndpointUrl = endpointUrl, + SecurityMode = MessageSecurityMode.None, + SecurityPolicyUri = SecurityPolicies.None, Server = new ApplicationDescription { - ApplicationUri = applicationUri + ApplicationUri = applicationUri, + DiscoveryUrls = new ArrayOf(new[] { endpointUrl }) } }; return new ConfiguredEndpoint(null, description, configuration: null); } - /// - /// Creates a mock that returns the given - /// redundancy support and service level from ReadAsync, - /// and optionally returns a redundant server array from - /// ReadValueAsync. - /// private static Mock CreateMockSession( int redundancySupport, byte serviceLevel, - RedundantServerDataType[]? redundantServers = null) + RedundantServerDataType[]? redundantServers = null, + string[]? serverUris = null, + string currentServerId = "") { var mock = new Mock(); + mock.SetupGet(s => s.ConfiguredEndpoint) + .Returns(CreateCurrentEndpoint("urn:current")); - // ReadValuesAsync reads two nodes via ReadAsync: RedundancySupport + ServiceLevel mock.Setup(s => s.ReadAsync( It.IsAny(), It.IsAny(), It.IsAny(), - It.Is>(r => r.Count == 2), + It.Is>(r => r.Count == 5), It.IsAny())) .ReturnsAsync(new ReadResponse { Results = [ new DataValue(new Variant(redundancySupport), StatusCodes.Good), - new DataValue(new Variant(serviceLevel), StatusCodes.Good) + new DataValue(new Variant(serviceLevel), StatusCodes.Good), + redundantServers == null + ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) + : CreateRedundantServerArrayValue(redundantServers), + serverUris == null + ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) + : new DataValue(new Variant(new ArrayOf(serverUris)), StatusCodes.Good), + new DataValue(new Variant(DateTime.MinValue), StatusCodes.Good) ], DiagnosticInfos = [] }); - // ReadValueAsync for the redundant server array (single-node read) - if (redundantServers != null) - { - ArrayOf extensionObjects = - Array.ConvertAll(redundantServers, s => new ExtensionObject(s)); - - mock.Setup(s => s.ReadAsync( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.Is>(r => r.Count == 1), - It.IsAny())) - .ReturnsAsync(new ReadResponse - { - Results = - [ - new DataValue( - new Variant(extensionObjects), - StatusCodes.Good) - ], - DiagnosticInfos = [] - }); - } + SetupSingleNodeRead( + mock, + VariableIds.Server_ServerRedundancy_CurrentServerId, + string.IsNullOrEmpty(currentServerId) + ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) + : new DataValue(new Variant(currentServerId), StatusCodes.Good)); return mock; } - /// - /// Creates a mock session where all reads return Bad status. - /// + private static void SetupSingleNodeRead( + Mock mock, + NodeId nodeId, + DataValue result) + { + mock.Setup(s => s.ReadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(r => r.Count == 1 && r[0].NodeId == nodeId), + It.IsAny())) + .ReturnsAsync(new ReadResponse + { + Results = [result], + DiagnosticInfos = [] + }); + } + + private static void VerifyBatchedRedundancyRead(Mock mock) + { + mock.Verify(s => s.ReadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(r => r.Count == 5), + It.IsAny()), Times.Once); + mock.Verify(s => s.ReadAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(r => + r.Count == 1 && + (r[0].NodeId == VariableIds.Server_ServerRedundancy_RedundantServerArray || + r[0].NodeId == VariableIds.Server_ServerRedundancy_ServerUriArray)), + It.IsAny()), Times.Never); + } + + private static DataValue CreateRedundantServerArrayValue( + RedundantServerDataType[] redundantServers) + { + ArrayOf extensionObjects = + Array.ConvertAll(redundantServers, s => new ExtensionObject(s)); + + return new DataValue(new Variant(extensionObjects), StatusCodes.Good); + } + private static Mock CreateMockSessionWithBadStatus() { var mock = new Mock(); @@ -442,12 +742,15 @@ private static Mock CreateMockSessionWithBadStatus() It.IsAny(), It.IsAny(), It.IsAny(), - It.Is>(r => r.Count == 2), + It.Is>(r => r.Count == 5), It.IsAny())) .ReturnsAsync(new ReadResponse { Results = [ + new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), + new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), + new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) ], @@ -456,5 +759,29 @@ private static Mock CreateMockSessionWithBadStatus() return mock; } + + private static Mock CreateMockSessionWithThrowingOptionalReads() + { + Mock mock = CreateMockSession( + redundancySupport: (int)RedundancySupport.Hot, + serviceLevel: ServiceLevels.NoData); + + return mock; + } + + private sealed class FixedTimeProvider : TimeProvider + { + public FixedTimeProvider(DateTime utcNow) + { + m_utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() + { + return m_utcNow; + } + + private readonly DateTimeOffset m_utcNow; + } } } diff --git a/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/McpServerOptionsTests.cs b/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/McpServerOptionsTests.cs index 978d1902ca..fbfa55391d 100644 --- a/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/McpServerOptionsTests.cs +++ b/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/McpServerOptionsTests.cs @@ -256,8 +256,12 @@ private static Assembly LoadMcpAssembly() : null; } - Assert.That(assemblyPath, Is.Not.Null.And.Not.Empty); - Assert.That(File.Exists(assemblyPath), Is.True); + if (string.IsNullOrEmpty(assemblyPath) || !File.Exists(assemblyPath)) + { + Assert.Ignore( + "The net10.0 Opc.Ua.Mcp assembly is not built for this CI leg " + + "(the MCP server only targets net10.0); skipping the reflective MCP server test."); + } return Assembly.LoadFrom(assemblyPath!); } diff --git a/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/PacketDecodePathValidationTests.cs b/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/PacketDecodePathValidationTests.cs index be3bca815d..82306a66e0 100644 --- a/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/PacketDecodePathValidationTests.cs +++ b/Tests/Opc.Ua.Core.Diagnostics.Tests/McpServerTools/PacketDecodePathValidationTests.cs @@ -180,8 +180,12 @@ private static Assembly LoadMcpAssembly() : null; } - Assert.That(assemblyPath, Is.Not.Null.And.Not.Empty); - Assert.That(File.Exists(assemblyPath), Is.True); + if (string.IsNullOrEmpty(assemblyPath) || !File.Exists(assemblyPath)) + { + Assert.Ignore( + "The net10.0 Opc.Ua.Mcp assembly is not built for this CI leg " + + "(the MCP server only targets net10.0); skipping the reflective MCP server test."); + } return Assembly.LoadFrom(assemblyPath!); } diff --git a/Tests/Opc.Ua.Core.Tests/Types/Constants/ServiceLevelsTests.cs b/Tests/Opc.Ua.Core.Tests/Types/Constants/ServiceLevelsTests.cs new file mode 100644 index 0000000000..4f2d8c690f --- /dev/null +++ b/Tests/Opc.Ua.Core.Tests/Types/Constants/ServiceLevelsTests.cs @@ -0,0 +1,87 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using NUnit.Framework; + +namespace Opc.Ua.Core.Tests.Types.Constants +{ + /// + /// Tests for the OPC UA ServiceLevel subrange helpers. + /// + [TestFixture] + [Category("ServiceLevel")] + [Parallelizable] + public sealed class ServiceLevelsTests + { + /// + /// Verifies the maintenance and no-data singleton boundaries. + /// + [Test] + public void GetSubrangeClassifiesMaintenanceAndNoDataBoundaries() + { + Assert.That(ServiceLevels.GetSubrange(ServiceLevels.Maintenance), Is.EqualTo(ServiceLevelSubrange.Maintenance)); + Assert.That(ServiceLevels.IsMaintenance(ServiceLevels.Maintenance), Is.True); + Assert.That(ServiceLevels.GetSubrange(ServiceLevels.NoData), Is.EqualTo(ServiceLevelSubrange.NoData)); + Assert.That(ServiceLevels.IsNoData(ServiceLevels.NoData), Is.True); + Assert.That(ServiceLevels.IsOperational(ServiceLevels.NoData), Is.False); + } + + /// + /// Verifies degraded subrange boundaries. + /// + [Test] + public void GetSubrangeClassifiesDegradedBoundaries() + { + Assert.That( + ServiceLevels.GetSubrange(ServiceLevels.DegradedMinimum), + Is.EqualTo(ServiceLevelSubrange.Degraded)); + Assert.That( + ServiceLevels.GetSubrange(ServiceLevels.DegradedMaximum), + Is.EqualTo(ServiceLevelSubrange.Degraded)); + Assert.That(ServiceLevels.IsDegraded(ServiceLevels.DegradedMinimum), Is.True); + Assert.That(ServiceLevels.IsOperational(ServiceLevels.DegradedMaximum), Is.True); + } + + /// + /// Verifies healthy subrange boundaries. + /// + [Test] + public void GetSubrangeClassifiesHealthyBoundaries() + { + Assert.That( + ServiceLevels.GetSubrange(ServiceLevels.HealthyMinimum), + Is.EqualTo(ServiceLevelSubrange.Healthy)); + Assert.That( + ServiceLevels.GetSubrange(ServiceLevels.Maximum), + Is.EqualTo(ServiceLevelSubrange.Healthy)); + Assert.That(ServiceLevels.IsHealthy(ServiceLevels.Maximum), Is.True); + Assert.That(ServiceLevels.IsOperational(ServiceLevels.HealthyMinimum), Is.True); + } + } +} diff --git a/Tests/Opc.Ua.InformationModel.Tests/RedundancyModelTests.cs b/Tests/Opc.Ua.InformationModel.Tests/RedundancyModelTests.cs index 9791160f15..d256c447e6 100644 --- a/Tests/Opc.Ua.InformationModel.Tests/RedundancyModelTests.cs +++ b/Tests/Opc.Ua.InformationModel.Tests/RedundancyModelTests.cs @@ -168,9 +168,22 @@ public async Task ServerRedundancyHasTypeDefinitionAsync() response.Results[0].References[0].NodeId, Session.NamespaceUris); - // Should be ServerRedundancyType or a subtype Assert.That(typeDefId, Is.Not.Null, "Type definition NodeId should not be null."); + + DataValue redundancySupport = await ReadValueAsync( + VariableIds.Server_ServerRedundancy_RedundancySupport) + .ConfigureAwait(false); + + var expectedTypeDefinition = redundancySupport.WrappedValue.GetInt32() switch + { + 4 => TransparentRedundancyTypeId, + 1 or 2 or 3 or 5 => NonTransparentRedundancyTypeId, + _ => ServerRedundancyTypeId + }; + + Assert.That(typeDefId, Is.EqualTo(expectedTypeDefinition), + "ServerRedundancy type definition must match the configured redundancy mode."); } [Test] @@ -204,6 +217,144 @@ public async Task CurrentServerIdExistsIfRedundancyEnabledAsync() "CurrentServerId should exist when redundancy is enabled."); } + [Test] + public async Task CurrentServerIdNodeIdMatchesStandardAsync() + { + DataValue dv = await ReadValueAsync( + VariableIds.Server_ServerRedundancy_RedundancySupport) + .ConfigureAwait(false); + + int redundancySupport = dv.WrappedValue.GetInt32(); + if (redundancySupport == 0) + { + Assert.Ignore("RedundancySupport is None; CurrentServerId not required."); + } + + DataValue currentServerId = await ReadAttributeAsync( + VariableIds.Server_ServerRedundancy_CurrentServerId, + Attributes.NodeId).ConfigureAwait(false); + + Assert.That(StatusCode.IsGood(currentServerId.StatusCode), Is.True); + NodeId nodeId = currentServerId.WrappedValue.GetNodeId(); + Assert.That(nodeId, Is.EqualTo(VariableIds.Server_ServerRedundancy_CurrentServerId)); + } + + [Test] + public async Task ServerUriArrayNodeIdMatchesStandardAsync() + { + DataValue dv = await ReadValueAsync( + VariableIds.Server_ServerRedundancy_RedundancySupport) + .ConfigureAwait(false); + + int redundancySupport = dv.WrappedValue.GetInt32(); + if (redundancySupport == 0) + { + Assert.Ignore("RedundancySupport is None; ServerUriArray not required."); + } + + DataValue serverUriArray = await ReadAttributeAsync( + VariableIds.Server_ServerRedundancy_ServerUriArray, + Attributes.NodeId).ConfigureAwait(false); + + Assert.That(StatusCode.IsGood(serverUriArray.StatusCode), Is.True); + NodeId nodeId = serverUriArray.WrappedValue.GetNodeId(); + Assert.That(nodeId, Is.EqualTo(VariableIds.Server_ServerRedundancy_ServerUriArray)); + } + + [Test] + public async Task RequestServerStateChangeMethodIsCallableAsync() + { + BrowseResult result = await BrowseChildrenAsync(ObjectIds.Server).ConfigureAwait(false); + + bool found = false; + foreach (ReferenceDescription rd in result.References) + { + if (rd.BrowseName.Name == "RequestServerStateChange") + { + found = true; + break; + } + } + + if (!found) + { + Assert.Ignore("RequestServerStateChange method not exposed."); + } + + DataValue executableAttr = await ReadAttributeAsync( + MethodIds.Server_RequestServerStateChange, + Attributes.Executable).ConfigureAwait(false); + + Assert.That(StatusCode.IsGood(executableAttr.StatusCode), Is.True); + } + + [Test] + public async Task RequestServerStateChangeHasCorrectSignatureAsync() + { + BrowseResult result = await BrowseChildrenAsync(ObjectIds.Server).ConfigureAwait(false); + + bool found = false; + foreach (ReferenceDescription rd in result.References) + { + if (rd.BrowseName.Name == "RequestServerStateChange") + { + found = true; + break; + } + } + + if (!found) + { + Assert.Ignore("RequestServerStateChange method not exposed."); + } + + ReadResponse response = await Session.ReadAsync( + null, 0, TimestampsToReturn.Both, + new ReadValueId[] + { + new() { + NodeId = MethodIds.Server_RequestServerStateChange, + AttributeId = Attributes.NodeClass + } + }.ToArrayOf(), + CancellationToken.None).ConfigureAwait(false); + + Assert.That(response.Results.Count, Is.EqualTo(1)); + Assert.That(StatusCode.IsGood(response.Results[0].StatusCode), Is.True); + NodeClass nodeClass = (NodeClass)response.Results[0].WrappedValue.GetInt32(); + Assert.That(nodeClass, Is.EqualTo(NodeClass.Method)); + } + + [Test] + public async Task RedundancySupportHasCorrectAccessLevelAsync() + { + DataValue accessLevel = await ReadAttributeAsync( + VariableIds.Server_ServerRedundancy_RedundancySupport, + Attributes.AccessLevel).ConfigureAwait(false); + + Assert.That(StatusCode.IsGood(accessLevel.StatusCode), Is.True); + byte level = accessLevel.WrappedValue.GetByte(); + Assert.That(level & (byte)AccessLevels.CurrentRead, Is.Not.Zero, + "RedundancySupport must be readable."); + } + + private async Task ReadAttributeAsync(NodeId nodeId, uint attributeId) + { + ReadResponse response = await Session.ReadAsync( + null, 0, TimestampsToReturn.Both, + new ReadValueId[] + { + new() { + NodeId = nodeId, + AttributeId = attributeId + } + }.ToArrayOf(), + CancellationToken.None).ConfigureAwait(false); + + Assert.That(response.Results.Count, Is.EqualTo(1)); + return response.Results[0]; + } + private async Task ReadValueAsync(NodeId nodeId) { ReadResponse response = await Session.ReadAsync( @@ -242,5 +393,9 @@ private async Task BrowseChildrenAsync( Assert.That(response.Results.Count, Is.EqualTo(1)); return response.Results[0]; } + + private static readonly NodeId ServerRedundancyTypeId = new(2034); + private static readonly NodeId TransparentRedundancyTypeId = new(2036); + private static readonly NodeId NonTransparentRedundancyTypeId = new(2039); } } diff --git a/Tests/Opc.Ua.Redundancy.Client.Tests/ClientReplicaCoordinatorTests.cs b/Tests/Opc.Ua.Redundancy.Client.Tests/ClientReplicaCoordinatorTests.cs new file mode 100644 index 0000000000..d6cc7788d6 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Client.Tests/ClientReplicaCoordinatorTests.cs @@ -0,0 +1,148 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Tests; + +namespace Opc.Ua.Client.Redundancy.Tests +{ + /// + /// Unit tests for the client replica coordinator and builder seams that do not + /// require a live server. Full lifecycle (token reuse, failover, subscription + /// transfer) is exercised by the integration tests. + /// + [TestFixture] + [Category("ClientRedundancy")] + public sealed class ClientReplicaCoordinatorTests + { + private ITelemetryContext m_telemetry = null!; + + [SetUp] + public void SetUp() + { + m_telemetry = NUnitTelemetryContext.Create(); + } + + [Test] + public void FailClosedRejectsNetworkedStoreWithoutProtector() + { + var options = new ClientReplicaOptions { CreateSessionAsync = _ => default }; + var election = new StaticLeaderElection(false); + var store = new FakeNetworkedStore(); + Assert.That( + () => new ClientReplicaCoordinator( + options, election, store, NullRecordProtector.Instance, m_telemetry), + Throws.InvalidOperationException); + } + + [Test] + public void InMemoryStoreWithoutProtectorIsAllowed() + { + var options = new ClientReplicaOptions { CreateSessionAsync = _ => default }; + var election = new StaticLeaderElection(false); + using var store = new InMemorySharedKeyValueStore(); + Assert.That( + () => new ClientReplicaCoordinator( + options, election, store, NullRecordProtector.Instance, m_telemetry), + Throws.Nothing); + } + + [Test] + public void MissingSessionFactoryThrows() + { + var election = new StaticLeaderElection(false); + using var store = new InMemorySharedKeyValueStore(); + Assert.That( + () => new ClientReplicaCoordinator( + new ClientReplicaOptions(), election, store, NullRecordProtector.Instance, m_telemetry), + Throws.ArgumentException); + } + + [Test] + public async Task ColdStandbyDoesNotConnectBeforeLeadershipAsync() + { + int created = 0; + var options = new ClientReplicaOptions + { + Mode = ClientStandbyMode.Cold, + CreateSessionAsync = _ => { created++; return default; } + }; + var election = new StaticLeaderElection(false); + using var store = new InMemorySharedKeyValueStore(); + await using var coordinator = new ClientReplicaCoordinator( + options, election, store, NullRecordProtector.Instance, m_telemetry); + await coordinator.StartAsync().ConfigureAwait(false); + Assert.That(created, Is.Zero); + Assert.That(coordinator.CurrentSession, Is.Null); + Assert.That(coordinator.IsLeader, Is.False); + } + + [Test] + public void BuilderRequiresRedundancySeams() + { + ClientReplicaSetBuilder builder = new ClientReplicaSetBuilder(m_telemetry) + .WithNodeId("a") + .WithStandbyMode(ClientStandbyMode.Hot) + .UseSession(_ => default); + Assert.That(() => builder.Build(), Throws.InvalidOperationException); + } + + private sealed class FakeNetworkedStore : ISharedKeyValueStore + { + public ValueTask<(bool Found, ByteString Value)> TryGetAsync(string key, CancellationToken ct = default) + => new((false, default)); + + public ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) => default; + + public ValueTask CompareAndSwapAsync( + string key, ByteString expected, ByteString value, CancellationToken ct = default) => new(true); + + public ValueTask DeleteAsync(string key, CancellationToken ct = default) => new(true); + + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, [EnumeratorCancellation] CancellationToken ct = default) + { + await Task.CompletedTask.ConfigureAwait(false); + yield break; + } + + public async IAsyncEnumerable WatchAsync( + string keyPrefix, [EnumeratorCancellation] CancellationToken ct = default) + { + await Task.CompletedTask.ConfigureAwait(false); + yield break; + } + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Client.Tests/CrdtClientKeyValueStoreTests.cs b/Tests/Opc.Ua.Redundancy.Client.Tests/CrdtClientKeyValueStoreTests.cs new file mode 100644 index 0000000000..37e53dd7af --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Client.Tests/CrdtClientKeyValueStoreTests.cs @@ -0,0 +1,59 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using NUnit.Framework; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Client.Redundancy.Tests +{ + /// + /// Coverage for the client-side CRDT shared key/value store. + /// + [TestFixture] + [Category("ClientRedundancy")] + public sealed class CrdtSharedKeyValueStoreTests + { + [Test] + public async Task SetThenGetRoundTripsAsync() + { + await using var network = new InMemoryNetwork(); + await using var store = new CrdtSharedKeyValueStore( + ReplicaId.New(), network.CreateTransport(), TimeProvider.System, CrdtReaderOptions.Default); + var value = new ByteString(new byte[] { 1, 2, 3, 4 }); + await store.SetAsync("session/a", value).ConfigureAwait(false); + (bool found, ByteString got) = await store.TryGetAsync("session/a").ConfigureAwait(false); + Assert.That(found, Is.True); + Assert.That(got.ToArray(), Is.EqualTo(new byte[] { 1, 2, 3, 4 })); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Client.Tests/Opc.Ua.Redundancy.Client.Tests.csproj b/Tests/Opc.Ua.Redundancy.Client.Tests/Opc.Ua.Redundancy.Client.Tests.csproj new file mode 100644 index 0000000000..3a2890dd36 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Client.Tests/Opc.Ua.Redundancy.Client.Tests.csproj @@ -0,0 +1,34 @@ + + + Exe + $(TestsTargetFrameworks) + Opc.Ua.Client.Redundancy.Tests + false + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + diff --git a/Tests/Opc.Ua.Redundancy.Client.Tests/Properties/AssemblyInfo.cs b/Tests/Opc.Ua.Redundancy.Client.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..2b9848014c --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Client.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +[assembly: CLSCompliant(false)] diff --git a/Tests/Opc.Ua.Redundancy.Client.Tests/RaftClientSharedStoreTests.cs b/Tests/Opc.Ua.Redundancy.Client.Tests/RaftClientSharedStoreTests.cs new file mode 100644 index 0000000000..05a89fe873 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Client.Tests/RaftClientSharedStoreTests.cs @@ -0,0 +1,111 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Client; + +namespace Opc.Ua.Client.Redundancy.Tests +{ + /// + /// Coverage for the Raft-backed client shared-store dependency-injection registrations. + /// + [TestFixture] + [Category("ClientRedundancy")] + public sealed class RaftClientSharedStoreTests + { + [Test] + public async Task AddRaftClientSharedStoreRegistersStoreAndElectionAsync() + { + var services = new ServiceCollection(); + services.AddRaftClientSharedStore(); + await using ServiceProvider provider = services.BuildServiceProvider(); + + ISharedKeyValueStore store = provider.GetRequiredService(); + ILeaderElection election = provider.GetRequiredService(); + + Assert.That(store, Is.InstanceOf()); + Assert.That(election, Is.InstanceOf()); + + bool created = await store.CompareAndSwapAsync( + "session/a", default, new ByteString(new byte[] { 1, 2, 3 })); + Assert.That(created, Is.True, "the client store provides a linearizable compare-and-swap"); + } + + [Test] + public async Task AddRedundantClientSharedStoreStrongUsesRaftStoreAsync() + { + var services = new ServiceCollection(); + services.AddRedundantClientSharedStore(RedundancyConsistencyMode.Strong, ReplicaId.New()); + await using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + } + + [Test] + public async Task AddRedundantClientSharedStoreEventualUsesHybridStoreAsync() + { + await using var network = new InMemoryNetwork(); + var services = new ServiceCollection(); + services.AddRedundantClientSharedStore( + RedundancyConsistencyMode.Eventual, ReplicaId.New(), _ => network.CreateTransport()); + await using ServiceProvider provider = services.BuildServiceProvider(); + + ISharedKeyValueStore store = provider.GetRequiredService(); + Assert.That(store, Is.InstanceOf()); + + // The strong-prefix keyspace gets linearizable CAS via Raft. + bool created = await store.CompareAndSwapAsync( + "nonce/x", default, new ByteString(new byte[] { 9 })); + Assert.That(created, Is.True); + } + + [Test] + public void AddRedundantClientSharedStoreEventualRequiresCrdtTransport() + { + var services = new ServiceCollection(); + + Assert.That( + () => services.AddRedundantClientSharedStore( + RedundancyConsistencyMode.Eventual, ReplicaId.New()), + Throws.TypeOf()); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/Client/KubernetesHttpApiClientCertificateTests.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/Client/KubernetesHttpApiClientCertificateTests.cs new file mode 100644 index 0000000000..724c7bdbe8 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/Client/KubernetesHttpApiClientCertificateTests.cs @@ -0,0 +1,153 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Net.Security; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using NUnit.Framework; + +namespace Opc.Ua.Redundancy.K8s.Tests +{ + /// + /// Unit tests for Kubernetes API server certificate validation. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class KubernetesHttpApiClientCertificateTests + { + [Test] + public void CertificateSignedByPinnedCaWithMatchingSanIsAccepted() + { + using X509Certificate2 root = CreateRootCertificate("Pinned Kubernetes CA"); + using X509Certificate2 server = CreateServerCertificate(root, "kubernetes.default.svc"); + using var chain = new X509Chain(); + + bool accepted = KubernetesHttpApiClient.ValidateServerCertificate( + root, + "kubernetes.default.svc", + server, + chain, + SslPolicyErrors.RemoteCertificateChainErrors); + + Assert.That(accepted, Is.True); + } + + [Test] + public void CertificateSignedByPinnedCaWithDifferentSanIsRejected() + { + using X509Certificate2 root = CreateRootCertificate("Pinned Kubernetes CA"); + using X509Certificate2 server = CreateServerCertificate(root, "attacker.default.svc"); + using var chain = new X509Chain(); + + bool accepted = KubernetesHttpApiClient.ValidateServerCertificate( + root, + "kubernetes.default.svc", + server, + chain, + SslPolicyErrors.RemoteCertificateChainErrors | SslPolicyErrors.RemoteCertificateNameMismatch); + + Assert.That(accepted, Is.False); + } + + [Test] + public void CertificateSignedByDifferentCaIsRejected() + { + using X509Certificate2 pinnedRoot = CreateRootCertificate("Pinned Kubernetes CA"); + using X509Certificate2 otherRoot = CreateRootCertificate("Other Kubernetes CA"); + using X509Certificate2 server = CreateServerCertificate(otherRoot, "kubernetes.default.svc"); + using var chain = new X509Chain(); + + bool accepted = KubernetesHttpApiClient.ValidateServerCertificate( + pinnedRoot, + "kubernetes.default.svc", + server, + chain, + SslPolicyErrors.RemoteCertificateChainErrors); + + Assert.That(accepted, Is.False); + } + + private static X509Certificate2 CreateRootCertificate(string commonName) + { + using RSA key = RSA.Create(2048); + var request = new CertificateRequest( + $"CN={commonName}", + key, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + request.CertificateExtensions.Add( + new X509BasicConstraintsExtension(certificateAuthority: true, hasPathLengthConstraint: false, 0, true)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.KeyCertSign | X509KeyUsageFlags.CrlSign, true)); + request.CertificateExtensions.Add( + new X509SubjectKeyIdentifierExtension(request.PublicKey, false)); + + DateTimeOffset notBefore = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset notAfter = DateTimeOffset.UtcNow.AddDays(30); + return request.CreateSelfSigned(notBefore, notAfter); + } + + private static X509Certificate2 CreateServerCertificate( + X509Certificate2 issuer, + string dnsName) + { + using RSA key = RSA.Create(2048); + var request = new CertificateRequest( + $"CN={dnsName}", + key, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName(dnsName); + request.CertificateExtensions.Add(sanBuilder.Build()); + request.CertificateExtensions.Add( + new X509BasicConstraintsExtension(certificateAuthority: false, hasPathLengthConstraint: false, 0, true)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.DigitalSignature | X509KeyUsageFlags.KeyEncipherment, true)); + request.CertificateExtensions.Add( + new X509EnhancedKeyUsageExtension( + new OidCollection + { + new Oid("1.3.6.1.5.5.7.3.1") + }, + false)); + + byte[] serialNumber = RandomNumberGenerator.GetBytes(16); + DateTimeOffset notBefore = DateTimeOffset.UtcNow.AddDays(-1); + DateTimeOffset notAfter = issuer.NotAfter.AddSeconds(-1); + return request.Create(issuer, notBefore, notAfter, serialNumber); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/Health/KubernetesReadinessServerTests.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/Health/KubernetesReadinessServerTests.cs new file mode 100644 index 0000000000..6afc21e6e6 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/Health/KubernetesReadinessServerTests.cs @@ -0,0 +1,56 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using NUnit.Framework; + +namespace Opc.Ua.Redundancy.K8s.Tests +{ + /// + /// Unit tests for Kubernetes readiness mapping. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class KubernetesReadinessServerTests + { + [Test] + public void HealthyServiceLevelIsReady() + { + Assert.That(KubernetesReadinessServer.IsReady(200, 200), Is.True); + Assert.That(KubernetesReadinessServer.IsReady(255, 200), Is.True); + } + + [Test] + public void MaintenanceAndNoDataAreNotReady() + { + Assert.That(KubernetesReadinessServer.IsReady(0, 200), Is.False); + Assert.That(KubernetesReadinessServer.IsReady(1, 200), Is.False); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/KubernetesRaftBuilderExtensionsTests.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/KubernetesRaftBuilderExtensionsTests.cs new file mode 100644 index 0000000000..e61a679b56 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/KubernetesRaftBuilderExtensionsTests.cs @@ -0,0 +1,153 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.K8s.Tests +{ + /// + /// Unit tests for — the multi-node RaftCs registration for a + /// Kubernetes StatefulSet. + /// + [TestFixture] + [Category("Distributed")] + public sealed class KubernetesRaftBuilderExtensionsTests + { + [Test] + public async Task RegistersRaftCsConsensusForStatefulSetAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseKubernetesRaftConsensus(options => + { + // Set the pod name explicitly so the test does not depend on the + // machine name; ordinal 1 → Raft node id 2 of a 3-node cluster. + options.PodName = "opcua-ha-1"; + options.HeadlessServiceName = "opcua-ha-headless"; + options.ReplicaCount = 3; + options.UseDurableStorage = false; + }); + + await using ServiceProvider provider = services.BuildServiceProvider(); + + IRaftConsensus consensus = provider.GetRequiredService(); + Assert.That(consensus, Is.InstanceOf()); + } + + [Test] + public async Task RegistersDurableFileStorageWithFqdnPeersAsync() + { + string dir = Path.Combine(Path.GetTempPath(), "opcua-raft-" + System.Guid.NewGuid().ToString("N")); + try + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseKubernetesRaftConsensus(options => + { + options.PodName = "opcua-ha-0"; + options.HeadlessServiceName = "opcua-ha-headless"; + options.Namespace = "opcua"; // fully-qualified peer DNS + options.ReplicaCount = 3; + options.UseDurableStorage = true; // FileRaftStorage on a temp dir + options.StoragePath = dir; + }); + + // Resolve (builds the WAL + bootstraps the ConfState) then dispose + // (closes the WAL) before deleting the directory. + { + await using ServiceProvider provider = services.BuildServiceProvider(); + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + } + + Assert.That(Directory.Exists(dir), Is.True, "the file WAL directory was created"); + } + finally + { + if (Directory.Exists(dir)) + { + Directory.Delete(dir, true); + } + } + } + + [Test] + public void RejectsNullBuilderAndInvalidOptions() + { + Assert.That( + () => ((IOpcUaServerBuilder)null!).UseKubernetesRaftConsensus(), + Throws.ArgumentNullException); + + var services = new ServiceCollection(); + IOpcUaServerBuilder builder = services.AddOpcUa().AddServer(_ => { }); + + Assert.That( + () => builder.UseKubernetesRaftConsensus(o => o.ReplicaCount = 3), + Throws.ArgumentException, "HeadlessServiceName is required"); + Assert.That( + () => builder.UseKubernetesRaftConsensus(o => o.HeadlessServiceName = "h"), + Throws.ArgumentException, "ReplicaCount must be >= 1"); + } + + [Test] + public void ResolvingRejectsNonOrdinalPodName() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseKubernetesRaftConsensus(options => + { + options.PodName = "no-ordinal-here"; + options.HeadlessServiceName = "h"; + options.ReplicaCount = 3; + options.UseDurableStorage = false; + }); + + using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That( + () => provider.GetRequiredService(), + Throws.InvalidOperationException); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/KubernetesServerBuilderExtensionsTests.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/KubernetesServerBuilderExtensionsTests.cs new file mode 100644 index 0000000000..5bad6edcc5 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/KubernetesServerBuilderExtensionsTests.cs @@ -0,0 +1,92 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.K8s.Tests +{ + /// + /// Unit tests for the Kubernetes server builder fluent registration + /// (). + /// + [TestFixture] + [Category("Distributed")] + public sealed class KubernetesServerBuilderExtensionsTests + { + [Test] + public async Task UseKubernetesFeaturesRegisterResolvableServicesAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseKubernetes() + .UseKubernetesLeaderElection(options => options.UseSharedStoreFallback = false) + .UseKubernetesPeerDiscovery() + .UseKubernetesReadiness(); + + await using ServiceProvider provider = services.BuildServiceProvider(); + + // Outside a cluster the factory yields a not-in-cluster client and the + // leader election falls back to a static non-leader, all without any IO. + IKubernetesApiClient apiClient = provider.GetRequiredService(); + Assert.That(apiClient.IsInCluster, Is.False); + Assert.That(provider.GetRequiredService(), Is.Not.Null); + Assert.That(provider.GetRequiredService(), Is.Not.Null); + Assert.That(provider.GetRequiredService(), Is.Not.Null); + Assert.That(provider.GetServices(), Is.Not.Empty); + } + + [Test] + public void BuilderExtensionsRejectNullBuilder() + { + Assert.That( + () => ((IOpcUaServerBuilder)null!).UseKubernetes(), + Throws.ArgumentNullException); + Assert.That( + () => ((IOpcUaServerBuilder)null!).UseKubernetesLeaderElection(), + Throws.ArgumentNullException); + Assert.That( + () => ((IOpcUaServerBuilder)null!).UseKubernetesPeerDiscovery(), + Throws.ArgumentNullException); + Assert.That( + () => ((IOpcUaServerBuilder)null!).UseKubernetesReadiness(), + Throws.ArgumentNullException); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs new file mode 100644 index 0000000000..b800f8b5b1 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs @@ -0,0 +1,303 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Moq; +using NUnit.Framework; + +namespace Opc.Ua.Redundancy.K8s.Tests +{ + /// + /// Unit tests for Kubernetes Lease leader election. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class KubernetesLeaseLeaderElectionTests + { + [Test] + public async Task MissingLeaseCreatesLeaseAndBecomesLeaderAsync() + { + var api = NewApi(); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:00Z")); + KubernetesLease? created = null; + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())) + .ReturnsAsync((KubernetesLease?)null); + api.Setup(x => x.CreateLeaseAsync("ns", It.IsAny(), It.IsAny())) + .Callback((_, lease, _) => created = lease) + .ReturnsAsync((string _, KubernetesLease lease, CancellationToken _) => lease); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(election.IsLeader, Is.True); + Assert.That(created?.Spec.HolderIdentity, Is.EqualTo("pod-a")); + Assert.That(created?.Spec.LeaseDurationSeconds, Is.EqualTo(30)); + } + + [Test] + public async Task SameHolderRenewsLeaseAsync() + { + var api = NewApi(); + var lease = HeldLease("pod-a", ParseUtc("2026-01-01T00:00:00Z")); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:10Z")); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ReturnsAsync(lease); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(lease.Spec.HolderIdentity, Is.EqualTo("pod-a")); + Assert.That(lease.Spec.LeaseTransitions, Is.Zero); + } + + [Test] + public async Task LiveForeignHolderKeepsReplicaFollowerAsync() + { + var api = NewApi(); + var lease = HeldLease("pod-b", ParseUtc("2026-01-01T00:00:00Z")); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:10Z")); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.False); + Assert.That(election.IsLeader, Is.False); + api.Verify(x => x.ReplaceLeaseAsync(It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny()), Times.Never); + } + + [Test] + public async Task ExpiredForeignHolderCanBeTakenOverAsync() + { + var api = NewApi(); + var lease = HeldLease("pod-b", ParseUtc("2026-01-01T00:00:00Z")); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:31Z")); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ReturnsAsync(lease); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(lease.Spec.HolderIdentity, Is.EqualTo("pod-a")); + Assert.That(lease.Spec.LeaseTransitions, Is.EqualTo(1)); + } + + [Test] + public async Task ConflictLosesLeadershipAsync() + { + var api = NewApi(); + var lease = HeldLease("pod-a", ParseUtc("2026-01-01T00:00:00Z")); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:10Z")); + api.SetupSequence(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())) + .ReturnsAsync((KubernetesLease?)null) + .ReturnsAsync(lease); + api.Setup(x => x.CreateLeaseAsync("ns", It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, KubernetesLease newLease, CancellationToken _) => newLease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ThrowsAsync(new HttpRequestException("conflict", null, HttpStatusCode.Conflict)); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + Assert.That(await election.TryAcquireOrRenewAsync(), Is.True); + Assert.That(await election.TryAcquireOrRenewAsync(), Is.False); + Assert.That(election.IsLeader, Is.False); + } + + [Test] + public async Task DisposeReleasesOwnedLeaseAsync() + { + var api = NewApi(); + var lease = HeldLease("pod-a", ParseUtc("2026-01-01T00:00:00Z")); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ReturnsAsync(lease); + + var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), new FakeTimeProvider()); + await election.DisposeAsync(); + + Assert.That(lease.Spec.HolderIdentity, Is.Null); + } + + [Test] + public async Task NotInClusterCannotAcquireLeadershipAsync() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(false); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), new FakeTimeProvider()); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.False); + Assert.That(election.IsLeader, Is.False); + } + + [Test] + public async Task EmptyLeaseHolderCanBeAcquiredAsync() + { + var api = NewApi(); + var lease = HeldLease(string.Empty, ParseUtc("2026-01-01T00:00:00Z")); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ReturnsAsync(lease); + + await using var election = new KubernetesLeaseLeaderElection( + api.Object, + NewOptions(), + new FakeTimeProvider(ParseUtc("2026-01-01T00:00:05Z"))); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(lease.Spec.HolderIdentity, Is.EqualTo("pod-a")); + } + + [Test] + public async Task MissingRenewTimeUsesExpiredFallbackAsync() + { + var api = NewApi(); + var lease = HeldLease("pod-b", ParseUtc("2026-01-01T00:00:00Z")); + lease.Spec.RenewTime = null; + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ReturnsAsync(lease); + + await using var election = new KubernetesLeaseLeaderElection( + api.Object, + NewOptions(), + new FakeTimeProvider(ParseUtc("2026-01-01T00:00:05Z"))); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(lease.Spec.HolderIdentity, Is.EqualTo("pod-a")); + } + + [Test] + public async Task StartIsIdempotentAndDisposeStopsRenewLoopAsync() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(false); + var options = NewOptions(); + options.RenewInterval = TimeSpan.FromMilliseconds(1); + + var election = new KubernetesLeaseLeaderElection(api.Object, options, new FakeTimeProvider()); + election.Start(); + election.Start(); + + await election.DisposeAsync(); + + Assert.That(election.IsLeader, Is.False); + } + + [Test] + public async Task DisposeIgnoresReleaseFailureAsync() + { + var api = NewApi(); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())) + .ThrowsAsync(new InvalidOperationException("transient")); + + var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), new FakeTimeProvider()); + + Assert.DoesNotThrowAsync(async () => await election.DisposeAsync().AsTask()); + } + + [Test] + public void StaticReadinessComparesServiceLevel() + { + Assert.That(KubernetesLeaseLeaderElection.IsReadyServiceLevel(200, 100), Is.True); + Assert.That(KubernetesLeaseLeaderElection.IsReadyServiceLevel(99, 100), Is.False); + } + + [Test] + public void PublicConstructorRejectsNullOptions() + { + Assert.Throws(() => new KubernetesLeaseLeaderElection(null!)); + } + + private static DateTimeOffset ParseUtc(string value) + { + return DateTimeOffset.Parse(value, CultureInfo.InvariantCulture); + } + + private static KubernetesLeaderElectionOptions NewOptions() + { + var options = new KubernetesLeaderElectionOptions + { + LeaseName = "opcua", + LeaseDuration = TimeSpan.FromSeconds(30), + RenewInterval = TimeSpan.FromSeconds(10) + }; + options.Kubernetes.Namespace = "ns"; + options.Kubernetes.NodeId = "pod-a"; + return options; + } + + private static KubernetesLease HeldLease(string holder, DateTimeOffset renewTime) + { + return new KubernetesLease + { + Metadata = new KubernetesObjectMetadata + { + Name = "opcua", + Namespace = "ns", + ResourceVersion = "1" + }, + Spec = new KubernetesLeaseSpec + { + HolderIdentity = holder, + LeaseDurationSeconds = 30, + AcquireTime = renewTime.ToString("O", CultureInfo.InvariantCulture), + RenewTime = renewTime.ToString("O", CultureInfo.InvariantCulture), + LeaseTransitions = 0 + } + }; + } + + private static Mock NewApi() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(true); + return api; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/Opc.Ua.Redundancy.K8s.Tests.csproj b/Tests/Opc.Ua.Redundancy.K8s.Tests/Opc.Ua.Redundancy.K8s.Tests.csproj new file mode 100644 index 0000000000..9a77c1f1c4 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/Opc.Ua.Redundancy.K8s.Tests.csproj @@ -0,0 +1,41 @@ + + + Exe + + net8.0;net10.0 + $(CustomTestTarget) + $(CustomTestTarget) + true + Opc.Ua.Redundancy.K8s.Tests + false + enable + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/PeerDiscovery/KubernetesPeerDiscoveryTests.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/PeerDiscovery/KubernetesPeerDiscoveryTests.cs new file mode 100644 index 0000000000..0db7edb3bc --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/PeerDiscovery/KubernetesPeerDiscoveryTests.cs @@ -0,0 +1,221 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; + +using Opc.Ua.Redundancy.Server; + +namespace Opc.Ua.Redundancy.K8s.Tests +{ + /// + /// Unit tests for Kubernetes peer discovery. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class KubernetesPeerDiscoveryTests + { + private static readonly string[] ExpectedPeerUris = ["opc.tcp://10.0.0.2:4840"]; + + [Test] + public void EndpointSlicesConvertReadyAddressesToPeerUris() + { + KubernetesEndpointSliceList slices = NewSlices(); + var options = NewOptions(); + + ArrayOf peers = KubernetesPeerDiscovery.ToPeerUris(slices, options); + + Assert.That(peers.Span.ToArray(), Is.EqualTo(ExpectedPeerUris)); + } + + [Test] + public async Task RefreshPublishesChangedPeersAsync() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(true); + api.Setup(x => x.ListEndpointSlicesAsync("ns", "svc", It.IsAny())) + .ReturnsAsync(NewSlices()); + var discovery = new KubernetesPeerDiscovery(api.Object, NewOptions()); + ArrayOf observed = ArrayOf.Empty; + discovery.PeerServerUrisChanged += peers => observed = peers; + + ArrayOf refreshed = await discovery.RefreshAsync(); + + Assert.That(refreshed.Span.ToArray(), Is.EqualTo(ExpectedPeerUris)); + Assert.That(observed.Span.ToArray(), Is.EqualTo(ExpectedPeerUris)); + } + + [Test] + public async Task NotInClusterRefreshReturnsEmptyAsync() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(false); + var discovery = new KubernetesPeerDiscovery(api.Object, NewOptions()); + + ArrayOf refreshed = await discovery.RefreshAsync(); + + Assert.That(refreshed.Count, Is.Zero); + } + + [Test] + public void EndpointSlicesUseFallbackPortWhenNoPortsExist() + { + var options = NewOptions(); + options.Port = 4841; + KubernetesEndpointSliceList slices = NewSlices(); + slices.Items[0].Ports.Clear(); + + ArrayOf peers = KubernetesPeerDiscovery.ToPeerUris(slices, options); + + Assert.That(peers.Span.ToArray(), Is.EqualTo(new[] { "opc.tcp://10.0.0.2:4841" })); + } + + [Test] + public void EndpointSlicesUseFirstPortWhenNamedPortMissing() + { + var options = NewOptions(); + options.PortName = "missing"; + KubernetesEndpointSliceList slices = NewSlices(); + + ArrayOf peers = KubernetesPeerDiscovery.ToPeerUris(slices, options); + + Assert.That(peers.Span.ToArray(), Is.EqualTo(ExpectedPeerUris)); + } + + [Test] + public async Task RefreshWithSamePeersDoesNotRaiseChangedEventAgainAsync() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(true); + api.Setup(x => x.ListEndpointSlicesAsync("ns", "svc", It.IsAny())) + .ReturnsAsync(NewSlices()); + var discovery = new KubernetesPeerDiscovery(api.Object, NewOptions()); + int changedCount = 0; + discovery.PeerServerUrisChanged += _ => changedCount++; + + await discovery.RefreshAsync(); + await discovery.RefreshAsync(); + + Assert.That(changedCount, Is.EqualTo(1)); + } + + [Test] + public async Task PopulateCopiesCurrentPeerUrisAsync() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(true); + api.Setup(x => x.ListEndpointSlicesAsync("ns", "svc", It.IsAny())) + .ReturnsAsync(NewSlices()); + var discovery = new KubernetesPeerDiscovery(api.Object, NewOptions()); + var options = new ServerRedundancyOptions(); + + await discovery.RefreshAsync(); + discovery.Populate(options); + + Assert.That(options.PeerServerUris.ToArray(), Is.EqualTo(ExpectedPeerUris)); + } + + [Test] + public void PopulateRejectsNullOptions() + { + var api = new Mock(MockBehavior.Strict); + api.SetupGet(x => x.IsInCluster).Returns(false); + var discovery = new KubernetesPeerDiscovery(api.Object, NewOptions()); + + Assert.Throws(() => discovery.Populate(null!)); + } + + [Test] + public void EndpointSlicesRejectNullInputs() + { + Assert.Throws(() => KubernetesPeerDiscovery.ToPeerUris(null!, NewOptions())); + Assert.Throws(() => KubernetesPeerDiscovery.ToPeerUris(NewSlices(), null!)); + } + + private static KubernetesPeerDiscoveryOptions NewOptions() + { + var options = new KubernetesPeerDiscoveryOptions + { + ServiceName = "svc", + LocalAddress = "10.0.0.1" + }; + options.Kubernetes.Namespace = "ns"; + return options; + } + + private static KubernetesEndpointSliceList NewSlices() + { + return new KubernetesEndpointSliceList + { + Items = + [ + new KubernetesEndpointSlice + { + Ports = + [ + new KubernetesEndpointPort + { + Name = "opcua-tcp", + Port = 4840 + } + ], + Endpoints = + [ + new KubernetesEndpoint + { + Addresses = ["10.0.0.1"] + }, + new KubernetesEndpoint + { + Addresses = ["10.0.0.2"] + }, + new KubernetesEndpoint + { + Addresses = ["10.0.0.3"], + Conditions = new KubernetesEndpointConditions + { + Ready = false + } + } + ] + } + ] + }; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.K8s.Tests/Properties/AssemblyInfo.cs b/Tests/Opc.Ua.Redundancy.K8s.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..6839e42afc --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.K8s.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +[assembly: CLSCompliant(false)] \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/AddressSpaceSynchronizerTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/AddressSpaceSynchronizerTests.cs new file mode 100644 index 0000000000..5b60ad77bc --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/AddressSpaceSynchronizerTests.cs @@ -0,0 +1,196 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Integration tests for running + /// two replicas (writer + reader) over one shared store. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class AddressSpaceSynchronizerTests + { + private const ushort NamespaceIndex = 1; + private IServiceMessageContext m_messageContext = null!; + private SystemContext m_systemContext = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:sync"); + m_messageContext = messageContext; + m_systemContext = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + } + + [Test] + public async Task WriterSeedsEmptyStoreAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new InMemoryNodeStateStore(kv, m_messageContext); + var writerSpace = new DictionaryAddressSpace(m_systemContext); + await writerSpace.AddOrUpdateNodeAsync(NewVariable("seed", 1.0)); + + await using var writer = new AddressSpaceSynchronizer(store, writerSpace, () => true); + await writer.SeedOrHydrateAsync(); + + IStoredNode? stored = await store.TryGetNodeAsync(new NodeId("seed", NamespaceIndex)); + (bool found, _) = await store.TryReadValueAsync(new NodeId("seed", NamespaceIndex)); + Assert.That(stored, Is.Not.Null); + Assert.That(found, Is.True); + } + + [Test] + public async Task ReaderDoesNotSeedEmptyStoreAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new InMemoryNodeStateStore(kv, m_messageContext); + var readerSpace = new DictionaryAddressSpace(m_systemContext); + await readerSpace.AddOrUpdateNodeAsync(NewVariable("local", 1.0)); + + await using var reader = new AddressSpaceSynchronizer(store, readerSpace, () => false); + await reader.SeedOrHydrateAsync(); + + IStoredNode? stored = await store.TryGetNodeAsync(new NodeId("local", NamespaceIndex)); + Assert.That(reader.IsWriter, Is.False); + Assert.That(stored, Is.Null, "a reader must never write to the shared store"); + } + + [Test] + public async Task WriterReplicatesTopologyAndValueToReaderAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var writerStore = new InMemoryNodeStateStore(kv, m_messageContext); + var readerStore = new InMemoryNodeStateStore(kv, m_messageContext); + + var writerSpace = new DictionaryAddressSpace(m_systemContext); + var readerSpace = new DictionaryAddressSpace(m_systemContext); + + BaseDataVariableState nodeX = NewVariable("X", 1.0); + await writerSpace.AddOrUpdateNodeAsync(nodeX); + + await using var writer = new AddressSpaceSynchronizer(writerStore, writerSpace, () => true); + await using var reader = new AddressSpaceSynchronizer(readerStore, readerSpace, () => false); + + await writer.SeedOrHydrateAsync(); + writer.Start(); + + await reader.SeedOrHydrateAsync(); + Assert.That(readerSpace.TryGetNode(nodeX.NodeId, out _), Is.True, "reader hydrated X from the store"); + reader.Start(); + + // Value change on the writer propagates to the reader. + Task valueApplied = WaitForInboundAsync( + reader, c => c.Kind == NodeStateChangeKind.Value && c.NodeId == nodeX.NodeId); + nodeX.Value = new Variant(42.0); + nodeX.ClearChangeMasks(m_systemContext, false); + await AwaitWithTimeoutAsync(valueApplied); + + Assert.That(readerSpace.TryGetNode(nodeX.NodeId, out NodeState? rx), Is.True); + Assert.That(((BaseDataVariableState)rx!).Value, Is.EqualTo(new Variant(42.0))); + + // Adding a node on the writer propagates to the reader. + BaseDataVariableState nodeY = NewVariable("Y", 7.0); + Task addApplied = WaitForInboundAsync( + reader, c => c.Kind == NodeStateChangeKind.Upsert && c.NodeId == nodeY.NodeId); + await writerSpace.AddOrUpdateNodeAsync(nodeY); + await AwaitWithTimeoutAsync(addApplied); + + Assert.That(readerSpace.TryGetNode(nodeY.NodeId, out _), Is.True, "reader received added node Y"); + + // Removing a node on the writer propagates to the reader. + Task deleteApplied = WaitForInboundAsync( + reader, c => c.Kind == NodeStateChangeKind.Delete && c.NodeId == nodeX.NodeId); + await writerSpace.RemoveNodeAsync(nodeX.NodeId); + await AwaitWithTimeoutAsync(deleteApplied); + + Assert.That(readerSpace.TryGetNode(nodeX.NodeId, out _), Is.False, "reader removed node X"); + } + + private BaseDataVariableState NewVariable(string id, double value) + { + return new BaseDataVariableState(null) + { + NodeId = new NodeId(id, NamespaceIndex), + BrowseName = new QualifiedName(id, NamespaceIndex), + DisplayName = new LocalizedText(id), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(value) + }; + } + + private static Task WaitForInboundAsync( + AddressSpaceSynchronizer synchronizer, + Func predicate) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + void Handler(NodeStateChange change) + { + if (predicate(change)) + { + synchronizer.InboundApplied -= Handler; + tcs.TrySetResult(true); + } + } + + synchronizer.InboundApplied += Handler; + return tcs.Task; + } + + private static async Task AwaitWithTimeoutAsync(Task task) + { + Task completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.That(completed, Is.SameAs(task), "replication did not complete within the timeout"); + await task; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/CustomNodeManagerAddressSpaceTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/CustomNodeManagerAddressSpaceTests.cs new file mode 100644 index 0000000000..dfae8f41d0 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/CustomNodeManagerAddressSpaceTests.cs @@ -0,0 +1,152 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System.Collections.Generic; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Tests the adapter exposed by + /// . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class CustomNodeManagerAddressSpaceTests + { + private const ushort NamespaceIndex = 1; + private const string NamespaceUri = "urn:test:custom-node-manager-address-space"; + + [Test] + public async Task CreateLocalAddressSpaceAdaptsPredefinedNodes() + { + Mock server = CreateServer(); + using var nodeManager = new TestNodeManager(server.Object, NamespaceUri); + BaseDataVariableState root = NewVariable("root", null); + BaseDataVariableState child = NewVariable("child", root); + BaseDataVariableState secondRoot = NewVariable("second-root", null); + root.AddChild(child); + nodeManager.AddPredefinedNodePublic(nodeManager.SystemContext, root); + nodeManager.AddPredefinedNodePublic(nodeManager.SystemContext, secondRoot); + + var source = (ILocalAddressSpaceSource)nodeManager; + ILocalAddressSpace addressSpace = source.CreateLocalAddressSpace(); + + Assert.That(addressSpace.Context, Is.SameAs(nodeManager.SystemContext)); + Assert.That( + addressSpace.Nodes, + Has.Exactly(1).Property(nameof(NodeState.NodeId)).EqualTo(root.NodeId)); + Assert.That( + addressSpace.Nodes, + Has.Exactly(1).Property(nameof(NodeState.NodeId)).EqualTo(secondRoot.NodeId)); + Assert.That(addressSpace.Nodes, Has.None.Property(nameof(NodeState.NodeId)).EqualTo(child.NodeId)); + Assert.That(addressSpace.TryGetNode(root.NodeId, out NodeState? found), Is.True); + Assert.That(found, Is.SameAs(root)); + + BaseDataVariableState added = NewVariable("added", null); + NodeState? addedEventNode = null; + addressSpace.NodeAdded += node => addedEventNode = node; + + await addressSpace.AddOrUpdateNodeAsync(added); + + Assert.That(addedEventNode, Is.SameAs(added)); + Assert.That(addressSpace.TryGetNode(added.NodeId, out NodeState? addedFound), Is.True); + Assert.That(addedFound, Is.SameAs(added)); + + NodeId? removedEventNodeId = null; + addressSpace.NodeRemoved += nodeId => removedEventNodeId = nodeId; + + bool removed = await addressSpace.RemoveNodeAsync(added.NodeId); + + Assert.That(removed, Is.True); + Assert.That(removedEventNodeId, Is.EqualTo(added.NodeId)); + Assert.That(addressSpace.TryGetNode(added.NodeId, out _), Is.False); + Assert.That(await addressSpace.RemoveNodeAsync(added.NodeId), Is.False); + } + + private static BaseDataVariableState NewVariable(string id, NodeState? parent) + { + return new BaseDataVariableState(parent) + { + NodeId = new NodeId(id, NamespaceIndex), + BrowseName = new QualifiedName(id, NamespaceIndex), + DisplayName = new LocalizedText(id), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(1.0) + }; + } + + private static Mock CreateServer() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + var namespaceUris = new NamespaceTable(); + namespaceUris.GetIndexOrAppend(NamespaceUri); + var serverUris = new StringTable(); + var server = new Mock(); + var masterNodeManager = new Mock(); + server.Setup(s => s.NamespaceUris).Returns(namespaceUris); + server.Setup(s => s.ServerUris).Returns(serverUris); + server.Setup(s => s.TypeTree).Returns(new TypeTable(namespaceUris)); + server.Setup(s => s.Factory).Returns(EncodeableFactory.Create()); + server.Setup(s => s.Telemetry).Returns(telemetry); + server.Setup(s => s.NodeManager).Returns(masterNodeManager.Object); + server.Setup(s => s.DefaultSystemContext).Returns(new ServerSystemContext(server.Object)); + return server; + } + + private sealed class TestNodeManager : CustomNodeManager2 + { + public TestNodeManager(IServerInternal server, params string[] namespaceUris) + : base(server, namespaceUris) + { + } + + public void AddPredefinedNodePublic(ISystemContext context, NodeState node) + { + AddPredefinedNode(context, node); + } + + public override void CreateAddressSpace(IDictionary> externalReferences) + { + } + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/DistributedAddressSpaceOptionsTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/DistributedAddressSpaceOptionsTests.cs new file mode 100644 index 0000000000..c9a93ff0d9 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/DistributedAddressSpaceOptionsTests.cs @@ -0,0 +1,94 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for distributed address-space fluent options. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class DistributedAddressSpaceOptionsTests + { + [Test] + public void DefaultsMatchStaticSingleLeaderDeployment() + { + var options = new DistributedAddressSpaceOptions(); + + Assert.That(options.KeyValueStoreFactory, Is.Null); + Assert.That(options.UseLeaderElection, Is.False); + Assert.That(options.LeaseKey, Is.EqualTo("addressspace/leader")); + Assert.That(options.NodeId, Is.EqualTo(Environment.MachineName)); + Assert.That(options.LeaseDuration, Is.EqualTo(TimeSpan.FromSeconds(30))); + Assert.That(options.RenewInterval, Is.EqualTo(TimeSpan.FromSeconds(10))); + Assert.That(options.RedundancyMode, Is.EqualTo(RedundancySupport.Warm)); + Assert.That((object?)options.ServiceLevelLoadMetric, Is.Null); + Assert.That((object?)options.HealthServiceLevel, Is.Null); + } + + [Test] + public async Task UseDistributedAddressSpaceRegistersDistributedServicesAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseDistributedAddressSpace(options => + { + options.RedundancyMode = RedundancySupport.Hot; + options.ServiceLevelLoadMetric = () => 2; + }); + await using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + ILeaderElection election = provider.GetRequiredService(); + Assert.That(election, Is.InstanceOf()); + Assert.That(election.IsLeader, Is.True); + + IServiceLevelProvider serviceLevelProvider = provider.GetRequiredService(); + Assert.That(serviceLevelProvider, Is.InstanceOf()); + Assert.That(serviceLevelProvider.GetServiceLevel(), Is.EqualTo((byte)(ServiceLevels.Maximum - 2))); + Assert.That( + provider.GetServices(), + Has.Some.InstanceOf()); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/DistributedAddressSpaceStartupTaskTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/DistributedAddressSpaceStartupTaskTests.cs new file mode 100644 index 0000000000..9cbbd88a72 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/DistributedAddressSpaceStartupTaskTests.cs @@ -0,0 +1,125 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Tests for : it wires a + /// synchronizer to every opted-in node manager and (as writer) seeds the + /// node manager's address space into the shared store. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class DistributedAddressSpaceStartupTaskTests + { + private const ushort NamespaceIndex = 1; + + [Test] + public async Task WiresSynchronizerAndSeedsOptedInNodeManagerAsync() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:wire"); + var systemContext = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + + // A node manager that opts in and exposes a one-node address space. + var addressSpace = new DictionaryAddressSpace(systemContext); + var nodeId = new NodeId("seeded", NamespaceIndex); + await addressSpace.AddOrUpdateNodeAsync(new BaseDataVariableState(null) + { + NodeId = nodeId, + BrowseName = new QualifiedName("Seeded", NamespaceIndex), + DisplayName = new LocalizedText("Seeded"), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(1.0) + }); + + var nodeManager = new Mock(); + nodeManager.As() + .Setup(s => s.CreateLocalAddressSpace()) + .Returns(addressSpace); + + var masterNodeManager = new Mock(); + masterNodeManager.Setup(m => m.NodeManagers).Returns(new[] { nodeManager.Object }); + + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(telemetry); + server.Setup(s => s.MessageContext).Returns(messageContext); + server.Setup(s => s.NamespaceUris).Returns(messageContext.NamespaceUris); + server.Setup(s => s.NodeManager).Returns(masterNodeManager.Object); + + using var kv = new InMemorySharedKeyValueStore(); + var election = new StaticLeaderElection(true); + var task = new DistributedAddressSpaceStartupTask(kv, election); + + await task.OnServerStartedAsync(server.Object); + + // The writer must have seeded the node into the shared store, and + // registered a default store in the task-owned registry. + var verifyStore = new InMemoryNodeStateStore(kv, messageContext); + IStoredNode? stored = await verifyStore.TryGetNodeAsync(nodeId); + + Assert.That(stored, Is.Not.Null, "writer should have seeded the opted-in node manager's address space"); + Assert.That(task.NodeStateStoreRegistry, Is.Not.Null); + Assert.That(task.NodeStateStoreRegistry!.Resolve(nodeId), Is.Not.Null, "a default node state store should be registered"); + + await task.DisposeAsync(); + } + + [Test] + public void ConstructorThrowsOnNullArguments() + { + using var kv = new InMemorySharedKeyValueStore(); + var election = new StaticLeaderElection(true); + + Assert.That(() => new DistributedAddressSpaceStartupTask(null!, election), Throws.ArgumentNullException); + Assert.That(() => new DistributedAddressSpaceStartupTask(kv, null!), Throws.ArgumentNullException); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/InMemoryNodeStateStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/InMemoryNodeStateStoreTests.cs new file mode 100644 index 0000000000..749d10b81f --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/InMemoryNodeStateStoreTests.cs @@ -0,0 +1,472 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for , including a real + /// NodeState binary round-trip through the store. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class InMemoryNodeStateStoreTests + { + private const ushort NamespaceIndex = 1; + private IServiceMessageContext m_messageContext; + private SystemContext m_systemContext; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:distributed"); + m_messageContext = messageContext; + m_systemContext = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + } + + [Test] + public async Task UpsertTryGetAndDeleteNodeAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var nodeId = new NodeId("var1", NamespaceIndex); + ByteString payload = ByteString.From(new byte[] { 1, 2, 3, 4 }); + + await store.UpsertNodeAsync(new StoredNode(nodeId, payload)); + IStoredNode? loaded = await store.TryGetNodeAsync(nodeId); + bool deleted = await store.DeleteNodeAsync(nodeId); + IStoredNode? afterDelete = await store.TryGetNodeAsync(nodeId); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded!.NodeId, Is.EqualTo(nodeId)); + Assert.That(loaded.Payload.ToArray(), Is.EqualTo(payload.ToArray())); + Assert.That(deleted, Is.True); + Assert.That(afterDelete, Is.Null); + } + + [Test] + public async Task EnumerateReturnsAllStoredNodesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var a = new NodeId("a", NamespaceIndex); + var b = new NodeId("b", NamespaceIndex); + await store.UpsertNodeAsync(new StoredNode(a, ByteString.From(new byte[] { 1 }))); + await store.UpsertNodeAsync(new StoredNode(b, ByteString.From(new byte[] { 2 }))); + + var ids = new System.Collections.Generic.List(); + await foreach (IStoredNode node in store.EnumerateAsync()) + { + ids.Add(node.NodeId); + } + + Assert.That(ids, Is.EquivalentTo(new[] { a, b })); + } + + [Test] + public async Task WriteAndReadValueRoundTripsAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var nodeId = new NodeId("v", NamespaceIndex); + var original = new DataValue(new Variant(42.0), StatusCodes.Good, DateTimeUtc.Now); + + await store.WriteValueAsync(nodeId, original); + (bool found, DataValue read) = await store.TryReadValueAsync(nodeId); + + Assert.That(found, Is.True); + Assert.That(read.WrappedValue, Is.EqualTo(original.WrappedValue)); + Assert.That(read.StatusCode, Is.EqualTo(original.StatusCode)); + Assert.That(read.SourceTimestamp, Is.EqualTo(original.SourceTimestamp)); + } + + [Test] + public async Task TryReadValueMissingReturnsFalseAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + + (bool found, DataValue read) = await store.TryReadValueAsync(new NodeId("nope", NamespaceIndex)); + + Assert.That(found, Is.False); + Assert.That(read.IsNull, Is.True); + } + + [Test] + public async Task EnumerateValuesReturnsAllStoredValuesInOnePassAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var a = new NodeId("a", NamespaceIndex); + var b = new NodeId("b", NamespaceIndex); + await store.WriteValueAsync(a, new DataValue(new Variant(1.0), StatusCodes.Good, DateTimeUtc.Now)); + await store.WriteValueAsync(b, new DataValue(new Variant(2.0), StatusCodes.Good, DateTimeUtc.Now)); + + var seen = new Dictionary(); + await foreach ((NodeId nodeId, DataValue value) in store.EnumerateValuesAsync()) + { + seen[nodeId] = value; + } + + Assert.That(seen.Keys, Is.EquivalentTo(new[] { a, b })); + Assert.That(seen[a].WrappedValue, Is.EqualTo(new Variant(1.0))); + Assert.That(seen[b].WrappedValue, Is.EqualTo(new Variant(2.0))); + } + + [Test] + public async Task EnumerateValuesRoundTripsThroughRecordProtectorAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(i + 1); + } + using var protector = new AesCbcHmacRecordProtector(key); + using var store = new InMemoryNodeStateStore(kv, m_messageContext, protector); + var nodeId = new NodeId("protected", NamespaceIndex); + await store.WriteValueAsync(nodeId, new DataValue(new Variant(7.0), StatusCodes.Good, DateTimeUtc.Now)); + + var seen = new Dictionary(); + await foreach ((NodeId id, DataValue value) in store.EnumerateValuesAsync()) + { + seen[id] = value; + } + + Assert.That(seen.Keys, Is.EquivalentTo(new[] { nodeId })); + Assert.That(seen[nodeId].WrappedValue, Is.EqualTo(new Variant(7.0))); + } + + [Test] + public async Task EnumerateValuesOnEmptyStoreYieldsNothingAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + + int count = 0; + await foreach ((NodeId _, DataValue _) in store.EnumerateValuesAsync()) + { + count++; + } + + Assert.That(count, Is.Zero); + } + + [Test] + public async Task WriteAndReadSnapshotRoundTripsEntriesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var a = new NodeId("a", NamespaceIndex); + var b = new NodeId("b", NamespaceIndex); + await store.UpsertNodeAsync(new StoredNode(a, ByteString.From(new byte[] { 1, 2 }))); + await store.UpsertNodeAsync(new StoredNode(b, ByteString.From(new byte[] { 3 }))); + await store.WriteValueAsync(a, new DataValue(new Variant(5.0), StatusCodes.Good, DateTimeUtc.Now)); + + await store.WriteSnapshotAsync(); + NodeStateSnapshot? snapshot = await store.TryReadSnapshotAsync(); + + Assert.That(snapshot, Is.Not.Null); + var upserts = new HashSet(); + var values = new Dictionary(); + await foreach (NodeStateChange entry in snapshot!.Entries) + { + if (entry.Kind == NodeStateChangeKind.Upsert) + { + upserts.Add(entry.NodeId); + } + else if (entry.Kind == NodeStateChangeKind.Value) + { + values[entry.NodeId] = entry.Value; + } + } + + Assert.That(upserts, Is.EquivalentTo(new[] { a, b })); + Assert.That(values.Keys, Is.EquivalentTo(new[] { a })); + Assert.That(values[a].WrappedValue, Is.EqualTo(new Variant(5.0))); + Assert.That(snapshot.Sequence, Is.GreaterThanOrEqualTo(3)); + } + + [Test] + public async Task TryReadSnapshotReturnsNullWhenNonePublishedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + + NodeStateSnapshot? snapshot = await store.TryReadSnapshotAsync(); + + Assert.That(snapshot, Is.Null); + } + + [Test] + public async Task DeltaLogReplaysOnlyChangesAfterSequenceAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var a = new NodeId("a", NamespaceIndex); + await store.UpsertNodeAsync(new StoredNode(a, ByteString.From(new byte[] { 1 }))); + ulong afterUpsert = store.CurrentSequence; + await store.WriteValueAsync(a, new DataValue(new Variant(9.0), StatusCodes.Good, DateTimeUtc.Now)); + + var replayed = new List(); + await foreach (NodeStateChange change in store.ReadDeltaLogAsync(afterUpsert)) + { + replayed.Add(change); + } + + Assert.That(replayed, Has.Count.EqualTo(1)); + Assert.That(replayed[0].Kind, Is.EqualTo(NodeStateChangeKind.Value)); + Assert.That(replayed[0].NodeId, Is.EqualTo(a)); + Assert.That(replayed[0].Sequence, Is.GreaterThan(afterUpsert)); + } + + [Test] + public async Task WriteSnapshotTrimsDeltaLogAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var a = new NodeId("a", NamespaceIndex); + await store.UpsertNodeAsync(new StoredNode(a, ByteString.From(new byte[] { 1 }))); + await store.WriteValueAsync(a, new DataValue(new Variant(2.0), StatusCodes.Good, DateTimeUtc.Now)); + + await store.WriteSnapshotAsync(); + + var replayed = new List(); + await foreach (NodeStateChange change in store.ReadDeltaLogAsync(0)) + { + replayed.Add(change); + } + + Assert.That(replayed, Is.Empty); + } + + [Test] + public void ObserveSequenceRaisesHighWaterMarkOnly() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + + store.ObserveSequence(50); + Assert.That(store.CurrentSequence, Is.EqualTo(50)); + store.ObserveSequence(10); + Assert.That(store.CurrentSequence, Is.EqualTo(50)); + } + + [Test] + public async Task SubscribeObservesTopologyAndValueChangesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + using var cts = new CancellationTokenSource(); + var nodeId = new NodeId("watched", NamespaceIndex); + + await using System.Collections.Generic.IAsyncEnumerator changes = + store.SubscribeChangesAsync(cts.Token).GetAsyncEnumerator(); + + ValueTask upsert = changes.MoveNextAsync(); + await store.UpsertNodeAsync(new StoredNode(nodeId, ByteString.From(new byte[] { 7 }))); + Assert.That(await upsert, Is.True); + Assert.That(changes.Current.Kind, Is.EqualTo(NodeStateChangeKind.Upsert)); + Assert.That(changes.Current.NodeId, Is.EqualTo(nodeId)); + Assert.That(changes.Current.Node, Is.Not.Null); + + ValueTask valueChange = changes.MoveNextAsync(); + await store.WriteValueAsync(nodeId, new DataValue(new Variant(1.0), StatusCodes.Good, DateTimeUtc.Now)); + Assert.That(await valueChange, Is.True); + Assert.That(changes.Current.Kind, Is.EqualTo(NodeStateChangeKind.Value)); + Assert.That(changes.Current.NodeId, Is.EqualTo(nodeId)); + + ValueTask delete = changes.MoveNextAsync(); + await store.DeleteNodeAsync(nodeId); + Assert.That(await delete, Is.True); + Assert.That(changes.Current.Kind, Is.EqualTo(NodeStateChangeKind.Delete)); + Assert.That(changes.Current.NodeId, Is.EqualTo(nodeId)); + + cts.Cancel(); + } + + [Test] + public async Task SubscribePollsWhenStoreHasNoWatchAsync() + { + // A CRDT store has no change-feed (WatchAsync throws NotSupported), + // so the standby subscription must fall back to scan-polling instead + // of silently stopping. + await using var network = new InMemoryNetwork(); + await using var crdt = new CrdtSharedKeyValueStore( + ReplicaId.New(), network.CreateTransport(), TimeProvider.System, CrdtReaderOptions.Default); + using var store = new InMemoryNodeStateStore( + crdt, m_messageContext, null, TimeSpan.FromMilliseconds(50)); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(15)); + var nodeId = new NodeId("polled", NamespaceIndex); + + await using System.Collections.Generic.IAsyncEnumerator changes = + store.SubscribeChangesAsync(cts.Token).GetAsyncEnumerator(); + + // Start the subscription so the baseline scan runs, then upsert a + // node; a subsequent poll observes it as an Upsert. + ValueTask upsert = changes.MoveNextAsync(); + await Task.Delay(120, cts.Token); + await store.UpsertNodeAsync(new StoredNode(nodeId, ByteString.From(new byte[] { 9 })), cts.Token); + + Assert.That(await upsert, Is.True); + Assert.That(changes.Current.Kind, Is.EqualTo(NodeStateChangeKind.Upsert)); + Assert.That(changes.Current.NodeId, Is.EqualTo(nodeId)); + + cts.Cancel(); + } + + [Test] + public async Task NodeStateBinaryRoundTripThroughStoreAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var store = new InMemoryNodeStateStore(kv, m_messageContext); + var nodeId = new NodeId("sensor", NamespaceIndex); + + var original = new BaseDataVariableState(null) + { + NodeId = nodeId, + BrowseName = new QualifiedName("Sensor", NamespaceIndex), + DisplayName = new LocalizedText("Sensor"), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(3.14) + }; + + using var stream = new MemoryStream(); + original.SaveAsBinary(m_systemContext, stream); + await store.UpsertNodeAsync(new StoredNode(nodeId, ByteString.From(stream.ToArray()))); + + IStoredNode? stored = await store.TryGetNodeAsync(nodeId); + Assert.That(stored, Is.Not.Null); + + var restored = new BaseDataVariableState(null); + using var loadStream = new MemoryStream(stored!.Payload.ToArray()); + restored.LoadAsBinary(m_systemContext, loadStream); + + Assert.That(restored.BrowseName, Is.EqualTo(original.BrowseName)); + Assert.That(restored.Value, Is.EqualTo(original.Value)); + } + + [Test] + public async Task ProtectedNodeAndValueRoundTripAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(11)); + using var store = new InMemoryNodeStateStore(kv, m_messageContext, protector); + var nodeId = new NodeId("secret", NamespaceIndex); + ByteString payload = ByteString.From(new byte[] { 9, 8, 7, 6 }); + + await store.UpsertNodeAsync(new StoredNode(nodeId, payload)); + await store.WriteValueAsync(nodeId, new DataValue(new Variant(99.0), StatusCodes.Good)); + + IStoredNode? node = await store.TryGetNodeAsync(nodeId); + (bool found, DataValue value) = await store.TryReadValueAsync(nodeId); + + // The bytes persisted in the backend must not be the plaintext. + (bool rawFound, ByteString raw) = await kv.TryGetAsync("n/" + nodeId); + Assert.That(rawFound, Is.True); + Assert.That(raw.ToArray(), Is.Not.EqualTo(payload.ToArray())); + + Assert.That(node, Is.Not.Null); + Assert.That(node!.Payload.ToArray(), Is.EqualTo(payload.ToArray())); + Assert.That(found, Is.True); + Assert.That(value.WrappedValue, Is.EqualTo(new Variant(99.0))); + } + + [Test] + public async Task TamperedProtectedNodeIsRejectedFailClosedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(12)); + using var store = new InMemoryNodeStateStore(kv, m_messageContext, protector); + var nodeId = new NodeId("tampered", NamespaceIndex); + + await store.UpsertNodeAsync(new StoredNode(nodeId, ByteString.From(new byte[] { 1, 2, 3 }))); + + // A compromised store / rogue replica forges the persisted record. + await kv.SetAsync("n/" + nodeId, ByteString.From(new byte[] { 66, 66, 66, 66, 66 })); + + IStoredNode? node = await store.TryGetNodeAsync(nodeId); + + Assert.That(node, Is.Null); + } + + [Test] + public async Task NodeProtectedUnderDifferentKeyIsRejectedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var writerProtector = new AesCbcHmacRecordProtector(MakeKey(13)); + using var readerProtector = new AesCbcHmacRecordProtector(MakeKey(14)); + var writer = new InMemoryNodeStateStore(kv, m_messageContext, writerProtector); + var reader = new InMemoryNodeStateStore(kv, m_messageContext, readerProtector); + var nodeId = new NodeId("crosskey", NamespaceIndex); + + await writer.UpsertNodeAsync(new StoredNode(nodeId, ByteString.From(new byte[] { 5, 5 }))); + + IStoredNode? node = await reader.TryGetNodeAsync(nodeId); + + Assert.That(node, Is.Null); + } + + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(seed + i); + } + return key; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/NodeStateSerializerTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/NodeStateSerializerTests.cs new file mode 100644 index 0000000000..6b8f4e83ef --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/NodeStateSerializerTests.cs @@ -0,0 +1,133 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class NodeStateSerializerTests + { + private const ushort NamespaceIndex = 1; + private SystemContext m_context = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:serializer"); + m_context = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + } + + [Test] + public void ObjectRoundTripsAsBaseObjectState() + { + var original = new BaseObjectState(null) + { + NodeId = new NodeId("obj", NamespaceIndex), + BrowseName = new QualifiedName("Obj", NamespaceIndex), + DisplayName = new LocalizedText("Obj") + }; + + ByteString payload = NodeStateSerializer.Serialize(m_context, original); + NodeState restored = NodeStateSerializer.Deserialize(m_context, payload); + + Assert.That(restored, Is.InstanceOf()); + Assert.That(restored.NodeClass, Is.EqualTo(NodeClass.Object)); + Assert.That(restored.BrowseName, Is.EqualTo(original.BrowseName)); + } + + [Test] + public void VariableRoundTripsWithValue() + { + var original = new BaseDataVariableState(null) + { + NodeId = new NodeId("var", NamespaceIndex), + BrowseName = new QualifiedName("Var", NamespaceIndex), + DisplayName = new LocalizedText("Var"), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(2.5) + }; + + ByteString payload = NodeStateSerializer.Serialize(m_context, original); + NodeState restored = NodeStateSerializer.Deserialize(m_context, payload); + + Assert.That(restored, Is.InstanceOf()); + Assert.That(restored.NodeClass, Is.EqualTo(NodeClass.Variable)); + var variable = (BaseDataVariableState)restored; + Assert.That(variable.Value, Is.EqualTo(new Variant(2.5))); + Assert.That(variable.DataType, Is.EqualTo(DataTypeIds.Double)); + } + + [Test] + public void MethodRoundTripsAsMethodState() + { + var original = new MethodState(null) + { + NodeId = new NodeId("m", NamespaceIndex), + BrowseName = new QualifiedName("DoIt", NamespaceIndex), + DisplayName = new LocalizedText("DoIt"), + Executable = true, + UserExecutable = true + }; + + ByteString payload = NodeStateSerializer.Serialize(m_context, original); + NodeState restored = NodeStateSerializer.Deserialize(m_context, payload); + + Assert.That(restored, Is.InstanceOf()); + Assert.That(((MethodState)restored).Executable, Is.True); + } + + [Test] + public void DeserializeTooShortPayloadThrows() + { + ByteString tooShort = ByteString.From(new byte[] { 1, 2 }); + + Assert.That( + () => NodeStateSerializer.Deserialize(m_context, tooShort), + Throws.TypeOf()); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/NodeStateStoreRegistryTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/NodeStateStoreRegistryTests.cs new file mode 100644 index 0000000000..6b37b88cf1 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/AddressSpace/NodeStateStoreRegistryTests.cs @@ -0,0 +1,142 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +// CS0618: ServiceMessageContext.GlobalContext is obsolete, but obsolete APIs +// are permitted in test code (repo rule) and the context is never used for +// encoding by these resolution-precedence tests. +// TODO: switch to a per-fixture ServiceMessageContext.CreateEmpty() context. +#pragma warning disable CS0618 + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for resolution + /// precedence (node, then namespace, then default). + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class NodeStateStoreRegistryTests + { + private const string NamespaceUri = "urn:test:registry"; + + [Test] + public void ResolveByNodeTakesPrecedenceOverNamespaceAndDefault() + { + var namespaceTable = new NamespaceTable(); + ushort index = namespaceTable.GetIndexOrAppend(NamespaceUri); + using var registry = new NodeStateStoreRegistry(namespaceTable); + + using var kv = new InMemorySharedKeyValueStore(); + var nodeStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + var namespaceStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + var defaultStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + + var nodeId = new NodeId(5, index); + registry.RegisterForNode(nodeId, nodeStore); + registry.RegisterForNamespace(NamespaceUri, namespaceStore); + registry.RegisterDefault(defaultStore); + + Assert.That(registry.Resolve(nodeId), Is.SameAs(nodeStore)); + } + + [Test] + public void ResolveByNamespaceWhenNoNodeBinding() + { + var namespaceTable = new NamespaceTable(); + ushort index = namespaceTable.GetIndexOrAppend(NamespaceUri); + using var registry = new NodeStateStoreRegistry(namespaceTable); + + using var kv = new InMemorySharedKeyValueStore(); + var namespaceStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + var defaultStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + registry.RegisterForNamespace(NamespaceUri, namespaceStore); + registry.RegisterDefault(defaultStore); + + Assert.That(registry.Resolve(new NodeId(99, index)), Is.SameAs(namespaceStore)); + } + + [Test] + public void ResolveFallsBackToDefault() + { + var namespaceTable = new NamespaceTable(); + using var registry = new NodeStateStoreRegistry(namespaceTable); + using var kv = new InMemorySharedKeyValueStore(); + var defaultStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + registry.RegisterDefault(defaultStore); + + Assert.That(registry.Resolve(new NodeId(1, 0)), Is.SameAs(defaultStore)); + } + + [Test] + public void ResolveReturnsNullWhenEmpty() + { + using var registry = new NodeStateStoreRegistry(new NamespaceTable()); + + Assert.That(registry.Resolve(new NodeId(1, 0)), Is.Null); + Assert.That(registry.Resolve(NodeId.Null), Is.Null); + } + + [Test] + public void UnregisterForNodeRemovesBinding() + { + var namespaceTable = new NamespaceTable(); + using var registry = new NodeStateStoreRegistry(namespaceTable); + using var kv = new InMemorySharedKeyValueStore(); + var nodeStore = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + var nodeId = new NodeId(7, 0); + registry.RegisterForNode(nodeId, nodeStore); + + bool removed = registry.UnregisterForNode(nodeId); + + Assert.That(removed, Is.True); + Assert.That(registry.Resolve(nodeId), Is.Null); + Assert.That(registry.Stores, Has.Count.EqualTo(0)); + } + + [Test] + public void StoresSnapshotContainsEveryRegisteredStore() + { + var namespaceTable = new NamespaceTable(); + using var registry = new NodeStateStoreRegistry(namespaceTable); + using var kv = new InMemorySharedKeyValueStore(); + var first = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + var second = new InMemoryNodeStateStore(kv, ServiceMessageContext.GlobalContext); + + registry.RegisterForNode(new NodeId(1, 0), first); + registry.RegisterDefault(second); + + Assert.That(registry.Stores, Is.EquivalentTo(new INodeStateStore[] { first, second })); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/KeyValueStore/InMemorySharedKeyValueStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/KeyValueStore/InMemorySharedKeyValueStoreTests.cs new file mode 100644 index 0000000000..a46b3d86fd --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/KeyValueStore/InMemorySharedKeyValueStoreTests.cs @@ -0,0 +1,184 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class InMemorySharedKeyValueStoreTests + { + [Test] + public async Task SetAndTryGetReturnsStoredValueAsync() + { + using var store = new InMemorySharedKeyValueStore(); + ByteString payload = ByteString.From(new byte[] { 1, 2, 3 }); + + await store.SetAsync("k1", payload); + (bool found, ByteString value) = await store.TryGetAsync("k1"); + + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(payload.ToArray())); + } + + [Test] + public async Task TryGetMissingKeyReturnsFalseAsync() + { + using var store = new InMemorySharedKeyValueStore(); + + (bool found, ByteString value) = await store.TryGetAsync("missing"); + + Assert.That(found, Is.False); + Assert.That(value.IsNull, Is.True); + } + + [Test] + public async Task CompareAndSwapCreatesWhenAbsentAsync() + { + using var store = new InMemorySharedKeyValueStore(); + ByteString value = ByteString.From(new byte[] { 9 }); + + bool created = await store.CompareAndSwapAsync("k", default, value); + bool createdAgain = await store.CompareAndSwapAsync("k", default, value); + + Assert.That(created, Is.True); + Assert.That(createdAgain, Is.False, "second create-if-absent must fail because the key now exists"); + } + + [Test] + public async Task CompareAndSwapSwapsWhenValueMatchesAsync() + { + using var store = new InMemorySharedKeyValueStore(); + ByteString first = ByteString.From(new byte[] { 1 }); + ByteString second = ByteString.From(new byte[] { 2 }); + await store.SetAsync("k", first); + + bool swapped = await store.CompareAndSwapAsync("k", first, second); + (bool found, ByteString value) = await store.TryGetAsync("k"); + + Assert.That(swapped, Is.True); + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(second.ToArray())); + } + + [Test] + public async Task CompareAndSwapFailsWhenValueMismatchAsync() + { + using var store = new InMemorySharedKeyValueStore(); + ByteString actual = ByteString.From(new byte[] { 1 }); + ByteString wrongExpected = ByteString.From(new byte[] { 7 }); + ByteString desired = ByteString.From(new byte[] { 2 }); + await store.SetAsync("k", actual); + + bool swapped = await store.CompareAndSwapAsync("k", wrongExpected, desired); + (bool found, ByteString value) = await store.TryGetAsync("k"); + + Assert.That(swapped, Is.False); + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(actual.ToArray()), "value must be unchanged on a failed CAS"); + } + + [Test] + public async Task DeleteRemovesKeyAsync() + { + using var store = new InMemorySharedKeyValueStore(); + await store.SetAsync("k", ByteString.From(new byte[] { 1 })); + + bool deleted = await store.DeleteAsync("k"); + bool deletedAgain = await store.DeleteAsync("k"); + (bool found, _) = await store.TryGetAsync("k"); + + Assert.That(deleted, Is.True); + Assert.That(deletedAgain, Is.False); + Assert.That(found, Is.False); + } + + [Test] + public async Task ScanReturnsMatchingPrefixOnlyAsync() + { + using var store = new InMemorySharedKeyValueStore(); + await store.SetAsync("a/1", ByteString.From(new byte[] { 1 })); + await store.SetAsync("a/2", ByteString.From(new byte[] { 2 })); + await store.SetAsync("b/1", ByteString.From(new byte[] { 3 })); + + var keys = new List(); + await foreach (KeyValuePair entry in store.ScanAsync("a/")) + { + keys.Add(entry.Key); + } + + Assert.That(keys, Is.EquivalentTo(new[] { "a/1", "a/2" })); + } + + [Test] + public async Task WatchObservesSetAndDeleteForPrefixAsync() + { + using var store = new InMemorySharedKeyValueStore(); + using var cts = new CancellationTokenSource(); + + // Driving the enumerator manually guarantees the watcher is + // registered (synchronous prefix of the iterator runs on the + // first MoveNextAsync) before any mutation is published — no + // delays, no flakiness. + await using IAsyncEnumerator enumerator = + store.WatchAsync("a/", cts.Token).GetAsyncEnumerator(); + + ValueTask first = enumerator.MoveNextAsync(); + await store.SetAsync("b/ignored", ByteString.From(new byte[] { 0 })); + await store.SetAsync("a/1", ByteString.From(new byte[] { 1 })); + + Assert.That(await first, Is.True); + Assert.That(enumerator.Current.Kind, Is.EqualTo(KeyValueChangeKind.Set)); + Assert.That(enumerator.Current.Key, Is.EqualTo("a/1")); + + ValueTask second = enumerator.MoveNextAsync(); + await store.DeleteAsync("a/1"); + + Assert.That(await second, Is.True); + Assert.That(enumerator.Current.Kind, Is.EqualTo(KeyValueChangeKind.Delete)); + Assert.That(enumerator.Current.Key, Is.EqualTo("a/1")); + + cts.Cancel(); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Opc.Ua.Redundancy.Server.Tests.csproj b/Tests/Opc.Ua.Redundancy.Server.Tests/Opc.Ua.Redundancy.Server.Tests.csproj new file mode 100644 index 0000000000..a8285ad49f --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Opc.Ua.Redundancy.Server.Tests.csproj @@ -0,0 +1,41 @@ + + + Exe + $(TestsTargetFrameworks) + Opc.Ua.Redundancy.Server.Tests + false + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Properties/AssemblyInfo.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..6839e42afc --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,32 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +using System; + +[assembly: CLSCompliant(false)] \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/HybridSharedKeyValueStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/HybridSharedKeyValueStoreTests.cs new file mode 100644 index 0000000000..d8973cafdd --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/HybridSharedKeyValueStoreTests.cs @@ -0,0 +1,188 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for : prefix routing between a bulk (CRDT) backend and a + /// strong (Raft) backend. Two instances stand in for the backends so + /// routing is directly observable. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class HybridSharedKeyValueStoreTests + { + [Test] + public async Task BulkKeyRoutesToBulkStoreAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong); + + await hybrid.SetAsync("node/1", ByteString.From(new byte[] { 1 })); + + (bool inBulk, _) = await bulk.TryGetAsync("node/1"); + (bool inStrong, _) = await strong.TryGetAsync("node/1"); + Assert.That(inBulk, Is.True, "bulk keys live in the CRDT backend"); + Assert.That(inStrong, Is.False); + } + + [Test] + public async Task StrongKeyRoutesToStrongStoreAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong); + + await hybrid.SetAsync("nonce/abc", ByteString.From(new byte[] { 2 })); + + (bool inStrong, _) = await strong.TryGetAsync("nonce/abc"); + (bool inBulk, _) = await bulk.TryGetAsync("nonce/abc"); + Assert.That(inStrong, Is.True, "strong keys live in the Raft backend"); + Assert.That(inBulk, Is.False); + } + + [Test] + public async Task CompareAndSwapOnStrongKeyUsesStrongStoreAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong); + ByteString value = ByteString.From(new byte[] { 7 }); + + bool created = await hybrid.CompareAndSwapAsync("lease/leader", default, value); + (bool inStrong, _) = await strong.TryGetAsync("lease/leader"); + + Assert.That(created, Is.True); + Assert.That(inStrong, Is.True); + } + + [Test] + public async Task WatchOnStrongPrefixObservesStrongStoreAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + await using IAsyncEnumerator enumerator = + hybrid.WatchAsync("election/", cts.Token).GetAsyncEnumerator(); + + ValueTask first = enumerator.MoveNextAsync(); + await hybrid.SetAsync("election/leader", ByteString.From(new byte[] { 1 }), cts.Token); + + Assert.That(await first, Is.True); + Assert.That(enumerator.Current.Key, Is.EqualTo("election/leader")); + } + + [Test] + public async Task EmptyPrefixScanMergesBothStoresAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong); + + await hybrid.SetAsync("node/1", ByteString.From(new byte[] { 1 })); + await hybrid.SetAsync("nonce/a", ByteString.From(new byte[] { 2 })); + + var keys = new List(); + await foreach (KeyValuePair entry in hybrid.ScanAsync(string.Empty)) + { + keys.Add(entry.Key); + } + + Assert.That(keys, Is.EquivalentTo(new[] { "node/1", "nonce/a" })); + } + + [Test] + public async Task StrongPrefixScanReturnsStrongKeysOnlyAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong); + + await hybrid.SetAsync("node/1", ByteString.From(new byte[] { 1 })); + await hybrid.SetAsync("nonce/a", ByteString.From(new byte[] { 2 })); + await hybrid.SetAsync("nonce/b", ByteString.From(new byte[] { 3 })); + + var keys = new List(); + await foreach (KeyValuePair entry in hybrid.ScanAsync("nonce/")) + { + keys.Add(entry.Key); + } + + Assert.That(keys, Is.EquivalentTo(new[] { "nonce/a", "nonce/b" })); + } + + [Test] + public async Task OwnsStoresDisposesBothBackendsAsync() + { + var bulk = new InMemorySharedKeyValueStore(); + var strong = new InMemorySharedKeyValueStore(); + var hybrid = new HybridSharedKeyValueStore(bulk, strong, default, ownsStores: true); + + await hybrid.SetAsync("nonce/a", ByteString.From(new byte[] { 1 })); + await hybrid.DisposeAsync(); + + // InMemorySharedKeyValueStore.Dispose clears its data, so a hit here + // proves the dispose propagated to the owned backend. + (bool found, _) = await strong.TryGetAsync("nonce/a"); + Assert.That(found, Is.False); + } + + [Test] + public async Task CustomStrongPrefixReplacesDefaultsAsync() + { + using var bulk = new InMemorySharedKeyValueStore(); + using var strong = new InMemorySharedKeyValueStore(); + var prefixes = new ArrayOf(new[] { "cas/" }.AsMemory()); + await using var hybrid = new HybridSharedKeyValueStore(bulk, strong, prefixes); + + await hybrid.SetAsync("cas/x", ByteString.From(new byte[] { 1 })); + await hybrid.SetAsync("nonce/y", ByteString.From(new byte[] { 2 })); + + (bool casInStrong, _) = await strong.TryGetAsync("cas/x"); + (bool nonceInBulk, _) = await bulk.TryGetAsync("nonce/y"); + Assert.That(casInStrong, Is.True, "the custom prefix routes to the strong store"); + Assert.That(nonceInBulk, Is.True, "custom prefixes replace the defaults, so nonce/ is now a bulk key"); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/InProcessRaftConsensusTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/InProcessRaftConsensusTests.cs new file mode 100644 index 0000000000..1b17afa35e --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/InProcessRaftConsensusTests.cs @@ -0,0 +1,166 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for and . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class InProcessRaftConsensusTests + { + [Test] + public async Task SingleNodeBecomesLeaderOnStartAsync() + { + await using var node = new InProcessRaftConsensus(); + var transitions = new List(); + node.LeadershipChanged += value => transitions.Add(value); + + Assert.That(node.IsLeader, Is.False, "a node is not the leader before it starts"); + + await node.StartAsync(); + + Assert.That(node.IsLeader, Is.True); + Assert.That(transitions, Is.EqualTo(new[] { true })); + } + + [Test] + public async Task ProposeDeliversToCommittedInLogOrderAsync() + { + await using var node = new InProcessRaftConsensus(); + await node.StartAsync(); + + await node.ProposeAsync(new byte[] { 1 }); + await node.ProposeAsync(new byte[] { 2 }); + await node.ProposeAsync(new byte[] { 3 }); + + IReadOnlyList committed = await ReadCommittedAsync(node, 3); + + Assert.That(committed[0], Is.EqualTo(new byte[] { 1 })); + Assert.That(committed[1], Is.EqualTo(new byte[] { 2 })); + Assert.That(committed[2], Is.EqualTo(new byte[] { 3 })); + } + + [Test] + public async Task ClusterLowestIdIsLeaderAsync() + { + var cluster = new InProcessRaftCluster(); + await using InProcessRaftConsensus node1 = cluster.CreateNode(1); + await using InProcessRaftConsensus node2 = cluster.CreateNode(2); + + await node1.StartAsync(); + await node2.StartAsync(); + + Assert.That(node1.IsLeader, Is.True, "the lowest live id is the leader"); + Assert.That(node2.IsLeader, Is.False); + } + + [Test] + public async Task ClusterProposeBroadcastsToAllMembersInSameOrderAsync() + { + var cluster = new InProcessRaftCluster(); + await using InProcessRaftConsensus node1 = cluster.CreateNode(1); + await using InProcessRaftConsensus node2 = cluster.CreateNode(2); + await node1.StartAsync(); + await node2.StartAsync(); + + // Proposing on either member replicates the command to every member + // in one identical global order. + await node1.ProposeAsync(new byte[] { 0xA }); + await node2.ProposeAsync(new byte[] { 0xB }); + + IReadOnlyList seenBy1 = await ReadCommittedAsync(node1, 2); + IReadOnlyList seenBy2 = await ReadCommittedAsync(node2, 2); + + Assert.That(seenBy1[0], Is.EqualTo(new byte[] { 0xA })); + Assert.That(seenBy1[1], Is.EqualTo(new byte[] { 0xB })); + Assert.That(seenBy2[0], Is.EqualTo(new byte[] { 0xA })); + Assert.That(seenBy2[1], Is.EqualTo(new byte[] { 0xB })); + } + + [Test] + public async Task DisposingLeaderReelectsNextLowestAsync() + { + var cluster = new InProcessRaftCluster(); + InProcessRaftConsensus node1 = cluster.CreateNode(1); + await using InProcessRaftConsensus node2 = cluster.CreateNode(2); + + var node2Transitions = new List(); + node2.LeadershipChanged += value => node2Transitions.Add(value); + + await node1.StartAsync(); + await node2.StartAsync(); + Assert.That(node2.IsLeader, Is.False); + + await node1.DisposeAsync(); + + Assert.That(node2.IsLeader, Is.True, "the next-lowest live id takes over when the leader leaves"); + Assert.That(node2Transitions, Is.EqualTo(new[] { true })); + } + + [Test] + public async Task ProposeAfterDisposeThrowsAsync() + { + var node = new InProcessRaftConsensus(); + await node.StartAsync(); + await node.DisposeAsync(); + + Assert.That( + async () => await node.ProposeAsync(new byte[] { 1 }), + Throws.TypeOf()); + } + + private static async Task> ReadCommittedAsync( + InProcessRaftConsensus node, + int count) + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = new List(count); + for (int ii = 0; ii < count; ii++) + { + ReadOnlyMemory command = await node.Committed.ReadAsync(cts.Token); + result.Add(command.ToArray()); + } + return result; + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftCsClusterTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftCsClusterTests.cs new file mode 100644 index 0000000000..dfdfbef9a9 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftCsClusterTests.cs @@ -0,0 +1,242 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Raft; +using Raft.Transport; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Integration tests for a real multi-node cluster (RaftCs over an in-process + /// network): election, replication, follower forwarding, failover, and quorum-loss behaviour. + /// + [TestFixture] + [Category("Distributed")] + public sealed class RaftCsClusterTests + { + private static readonly ArrayOf MemberIds = new(new ulong[] { 1, 2, 3 }.AsMemory()); + + [Test] + public async Task ThreeNodeClusterConvergesViaFollowerWriteAsync() + { + await using var network = new InMemoryNetwork(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + RaftCsConsensus[] nodes = CreateNodes(network); + RaftSharedKeyValueStore[] stores = nodes + .Select(n => new RaftSharedKeyValueStore(n, ownsConsensus: false, commitTimeout: TimeSpan.FromSeconds(15))) + .ToArray(); + + try + { + // Nodes must start concurrently: each waits for an initial leader, + // which cannot be elected until a quorum is up. + await Task.WhenAll(nodes.Select(n => n.StartAsync(cts.Token).AsTask())); + + int leader = await WaitForLeaderAsync(nodes, cts.Token); + int follower = (leader + 1) % nodes.Length; + + // A write on a follower's store is forwarded to the leader, + // committed, and replicated to every replica. + ByteString payload = ByteString.From(new byte[] { 42 }); + await stores[follower].SetAsync("shared", payload, cts.Token); + + foreach (RaftSharedKeyValueStore store in stores) + { + ByteString observed = await WaitForValueAsync(store, "shared", cts.Token); + Assert.That(observed.ToArray(), Is.EqualTo(payload.ToArray())); + } + } + finally + { + await DisposeAllAsync(stores, nodes); + } + } + + [Test] + public async Task LeaderFailoverKeepsClusterWritableAsync() + { + await using var network = new InMemoryNetwork(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + RaftCsConsensus[] nodes = CreateNodes(network); + RaftSharedKeyValueStore[] stores = nodes + .Select(n => new RaftSharedKeyValueStore(n, ownsConsensus: false, commitTimeout: TimeSpan.FromSeconds(15))) + .ToArray(); + + try + { + await Task.WhenAll(nodes.Select(n => n.StartAsync(cts.Token).AsTask())); + int leader = await WaitForLeaderAsync(nodes, cts.Token); + + // Take the leader down; the remaining two must re-elect and stay + // writable (quorum 2 of the original 3). + await stores[leader].DisposeAsync(); + await nodes[leader].DisposeAsync(); + + int[] survivors = Enumerable.Range(0, nodes.Length).Where(i => i != leader).ToArray(); + await WaitForLeaderAsync(survivors.Select(i => nodes[i]).ToArray(), cts.Token); + + ByteString payload = ByteString.From(new byte[] { 7 }); + await stores[survivors[0]].SetAsync("after-failover", payload, cts.Token); + ByteString observed = await WaitForValueAsync(stores[survivors[1]], "after-failover", cts.Token); + Assert.That(observed.ToArray(), Is.EqualTo(payload.ToArray())); + } + finally + { + await DisposeAllAsync(stores, nodes); + } + } + + [Test] + public async Task ProposalTimesOutWhenQuorumLostAsync() + { + await using var network = new InMemoryNetwork(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + RaftCsConsensus[] nodes = CreateNodes(network); + // Short commit timeout so the quorum-loss failure is quick. + RaftSharedKeyValueStore[] stores = nodes + .Select(n => new RaftSharedKeyValueStore(n, ownsConsensus: false, commitTimeout: TimeSpan.FromSeconds(1))) + .ToArray(); + + try + { + await Task.WhenAll(nodes.Select(n => n.StartAsync(cts.Token).AsTask())); + int leader = await WaitForLeaderAsync(nodes, cts.Token); + + // Remove two of three replicas: the survivor can no longer reach a + // quorum, so a proposal fails via the commit timeout instead of + // hanging. + int[] toKill = Enumerable.Range(0, nodes.Length).Where(i => i != leader).ToArray(); + foreach (int i in toKill) + { + await stores[i].DisposeAsync(); + await nodes[i].DisposeAsync(); + } + + Assert.That( + async () => await stores[leader].SetAsync("no-quorum", ByteString.From(new byte[] { 1 }), cts.Token), + Throws.TypeOf()); + } + finally + { + await DisposeAllAsync(stores, nodes); + } + } + + private static RaftCsConsensus[] CreateNodes(InMemoryNetwork network) + { + var nodes = new RaftCsConsensus[3]; + for (int ii = 0; ii < nodes.Length; ii++) + { + ulong id = (ulong)(ii + 1); + nodes[ii] = RaftCsConsensus.CreateCluster( + id, + MemberIds, + network.CreateNode(id), + new RaftNodeOptions { TickInterval = TimeSpan.FromMilliseconds(10) }, + config => + { + config.ElectionTick = 10; + config.HeartbeatTick = 1; + config.PreVote = true; + // Distinct fixed timeouts make the lowest-id live node win, + // keeping the test deterministic. + config.RandomizedElectionTimeout = (int)(id * 6); + }, + readyTimeout: TimeSpan.FromSeconds(25)); + } + return nodes; + } + + private static async Task WaitForLeaderAsync(RaftCsConsensus[] nodes, CancellationToken ct) + { + while (true) + { + for (int ii = 0; ii < nodes.Length; ii++) + { + if (nodes[ii].IsLeader) + { + return ii; + } + } + await Task.Delay(20, ct); + } + } + + private static async Task WaitForValueAsync( + RaftSharedKeyValueStore store, + string key, + CancellationToken ct) + { + while (true) + { + (bool found, ByteString value) = await store.TryGetAsync(key, ct); + if (found) + { + return value; + } + await Task.Delay(10, ct); + } + } + + private static async Task DisposeAllAsync( + IReadOnlyList stores, + IReadOnlyList nodes) + { + foreach (RaftSharedKeyValueStore store in stores) + { + try + { + await store.DisposeAsync(); + } + catch (ObjectDisposedException) + { + // already disposed by the test body + } + } + foreach (RaftCsConsensus node in nodes) + { + await node.DisposeAsync(); + } + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftCsConsensusTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftCsConsensusTests.cs new file mode 100644 index 0000000000..b7d04a9ade --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftCsConsensusTests.cs @@ -0,0 +1,107 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Tests that exercise the in-repo Raft seams over a real (a single-node RaftCs + /// replica with real election, log replication, and commit), proving the adapter binding to the external engine. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class RaftCsConsensusTests + { + [Test] + public async Task SingleNodeElectsItselfLeaderAsync() + { + await using RaftCsConsensus consensus = RaftCsConsensus.CreateSingleNode(); + await using var election = new RaftLeaderElection(consensus); + + await consensus.StartAsync(); + + Assert.That(consensus.IsLeader, Is.True, "a single-voter RaftCs replica elects itself leader"); + Assert.That(election.IsLeader, Is.True); + } + + [Test] + public async Task StoreCompareAndSwapIsLinearizableAsync() + { + await using var store = new RaftSharedKeyValueStore( + RaftCsConsensus.CreateSingleNode(), ownsConsensus: true); + + bool created = await store.CompareAndSwapAsync("k", default, ByteString.From(new byte[] { 1 })); + bool createdAgain = await store.CompareAndSwapAsync("k", default, ByteString.From(new byte[] { 2 })); + (bool found, ByteString value) = await store.TryGetAsync("k"); + + Assert.That(created, Is.True); + Assert.That(createdAgain, Is.False, "the second create-if-absent loses once the key is committed"); + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(new byte[] { 1 })); + } + + [Test] + public async Task SetThenGetRoundTripsThroughCommittedLogAsync() + { + await using var store = new RaftSharedKeyValueStore( + RaftCsConsensus.CreateSingleNode(), ownsConsensus: true); + ByteString payload = ByteString.From(new byte[] { 4, 5, 6 }); + + await store.SetAsync("session/a", payload); + (bool found, ByteString value) = await store.TryGetAsync("session/a"); + + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(payload.ToArray())); + } + + [Test] + public async Task NonceRegistryEnforcesExactlyOnceAsync() + { + await using var store = new RaftSharedKeyValueStore( + RaftCsConsensus.CreateSingleNode(), ownsConsensus: true); + var registry = new SharedSingleUseNonceRegistry(store); + ByteString nonce = ByteString.From(new byte[] { 1, 2, 3, 4 }); + + bool first = await registry.TryConsumeAsync(nonce); + bool second = await registry.TryConsumeAsync(nonce); + + Assert.That(first, Is.True); + Assert.That(second, Is.False, "a replayed nonce is rejected by the RaftCs-backed registry"); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftLeaderElectionTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftLeaderElectionTests.cs new file mode 100644 index 0000000000..fb86a6fabd --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftLeaderElectionTests.cs @@ -0,0 +1,107 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for : leadership backed by native Raft consensus. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class RaftLeaderElectionTests + { + [Test] + public async Task SingleNodeElectionBecomesLeaderAsync() + { + await using var consensus = new InProcessRaftConsensus(); + await using var election = new RaftLeaderElection(consensus); + var transitions = new List(); + election.LeadershipChanged += value => transitions.Add(value); + + Assert.That(election.IsLeader, Is.False); + + election.Start(); + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(election.IsLeader, Is.True); + Assert.That(transitions, Is.EqualTo(new[] { true })); + } + + [Test] + public async Task FollowerIsNotLeaderAsync() + { + var cluster = new InProcessRaftCluster(); + await using InProcessRaftConsensus consensus1 = cluster.CreateNode(1); + await using InProcessRaftConsensus consensus2 = cluster.CreateNode(2); + await using var election1 = new RaftLeaderElection(consensus1); + await using var election2 = new RaftLeaderElection(consensus2); + + await consensus1.StartAsync(); + await consensus2.StartAsync(); + + Assert.That(election1.IsLeader, Is.True); + Assert.That(election2.IsLeader, Is.False); + } + + [Test] + public async Task ElectionFollowsConsensusFailoverAsync() + { + var cluster = new InProcessRaftCluster(); + InProcessRaftConsensus consensus1 = cluster.CreateNode(1); + await using InProcessRaftConsensus consensus2 = cluster.CreateNode(2); + await using var election2 = new RaftLeaderElection(consensus2); + + var node2Transitions = new List(); + election2.LeadershipChanged += value => node2Transitions.Add(value); + + await consensus1.StartAsync(); + await consensus2.StartAsync(); + Assert.That(election2.IsLeader, Is.False); + + // The leader leaves; election2 must observe the takeover. + await consensus1.DisposeAsync(); + + Assert.That(election2.IsLeader, Is.True, "the surviving replica is elected leader"); + Assert.That(node2Transitions, Is.EqualTo(new[] { true })); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftPrimitivesIntegrationTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftPrimitivesIntegrationTests.cs new file mode 100644 index 0000000000..c6c46b9092 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftPrimitivesIntegrationTests.cs @@ -0,0 +1,112 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Integration tests proving the linearizable primitives (single-use nonce registry, lease election) work over the + /// Raft store, and that the consistency-mode DI registration auto-wires them. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class RaftPrimitivesIntegrationTests + { + [Test] + public async Task NonceRegistryOverRaftEnforcesExactlyOnceUnderContentionAsync() + { + await using var store = new RaftSharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(store); + ByteString nonce = ByteString.From(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + const int contenders = 24; + + IEnumerable> races = Enumerable.Range(0, contenders) + .Select(_ => registry.TryConsumeAsync(nonce).AsTask()); + bool[] results = await Task.WhenAll(races); + + Assert.That(results.Count(consumed => consumed), Is.EqualTo(1), + "a single-use nonce may be consumed exactly once across the replica set"); + } + + [Test] + public async Task LeaseElectionOverRaftElectsSingleLeaderUnderContentionAsync() + { + await using var store = new RaftSharedKeyValueStore(); + await using var election1 = new SharedStoreLeaseElection( + store, "lease/leader", "node-1", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + await using var election2 = new SharedStoreLeaseElection( + store, "lease/leader", "node-2", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + + bool[] acquired = await Task.WhenAll( + election1.TryAcquireOrRenewAsync().AsTask(), + election2.TryAcquireOrRenewAsync().AsTask()); + + Assert.That(acquired.Count(won => won), Is.EqualTo(1), + "exactly one replica may hold the lease at a time"); + Assert.That(election1.IsLeader ^ election2.IsLeader, Is.True, "leadership is exclusive"); + } + + [Test] + public async Task ConsistencyRegistrationAutoWiresNonceRegistryAsync() + { + // The CRDT session factory resolves ISharedKeyValueStore and builds a + // SharedSingleUseNonceRegistry over it. With UseRedundancyConsistency + // that store is Raft-backed, so the nonce "nonce/" CAS is linearizable + // and no separate strongly-consistent backend (e.g. Redis) is needed. + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency(); + await using ServiceProvider provider = services.BuildServiceProvider(); + + ISharedKeyValueStore store = provider.GetRequiredService(); + var registry = new SharedSingleUseNonceRegistry(store); + ByteString nonce = ByteString.From(new byte[] { 9, 9, 9, 9 }); + + bool first = await registry.TryConsumeAsync(nonce); + bool second = await registry.TryConsumeAsync(nonce); + + Assert.That(first, Is.True); + Assert.That(second, Is.False, "a replayed nonce is rejected by the Raft-backed registry"); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftSharedKeyValueStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftSharedKeyValueStoreTests.cs new file mode 100644 index 0000000000..6fa87926b6 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RaftSharedKeyValueStoreTests.cs @@ -0,0 +1,281 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for : the linearizable CP store built on + /// . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class RaftSharedKeyValueStoreTests + { + [Test] + public async Task SetAndTryGetReturnsStoredValueAsync() + { + await using var store = new RaftSharedKeyValueStore(); + ByteString payload = ByteString.From(new byte[] { 1, 2, 3 }); + + await store.SetAsync("k1", payload); + (bool found, ByteString value) = await store.TryGetAsync("k1"); + + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(payload.ToArray())); + } + + [Test] + public async Task TryGetMissingKeyReturnsFalseAsync() + { + await using var store = new RaftSharedKeyValueStore(); + + (bool found, ByteString value) = await store.TryGetAsync("missing"); + + Assert.That(found, Is.False); + Assert.That(value.IsNull, Is.True); + } + + [Test] + public async Task CompareAndSwapCreatesWhenAbsentAsync() + { + await using var store = new RaftSharedKeyValueStore(); + ByteString value = ByteString.From(new byte[] { 9 }); + + bool created = await store.CompareAndSwapAsync("k", default, value); + bool createdAgain = await store.CompareAndSwapAsync("k", default, value); + + Assert.That(created, Is.True); + Assert.That(createdAgain, Is.False, "second create-if-absent must fail because the key now exists"); + } + + [Test] + public async Task CompareAndSwapSwapsWhenValueMatchesAsync() + { + await using var store = new RaftSharedKeyValueStore(); + ByteString first = ByteString.From(new byte[] { 1 }); + ByteString second = ByteString.From(new byte[] { 2 }); + await store.SetAsync("k", first); + + bool swapped = await store.CompareAndSwapAsync("k", first, second); + (bool found, ByteString value) = await store.TryGetAsync("k"); + + Assert.That(swapped, Is.True); + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(second.ToArray())); + } + + [Test] + public async Task CompareAndSwapFailsWhenValueMismatchAsync() + { + await using var store = new RaftSharedKeyValueStore(); + ByteString actual = ByteString.From(new byte[] { 1 }); + ByteString wrongExpected = ByteString.From(new byte[] { 7 }); + ByteString desired = ByteString.From(new byte[] { 2 }); + await store.SetAsync("k", actual); + + bool swapped = await store.CompareAndSwapAsync("k", wrongExpected, desired); + (bool found, ByteString value) = await store.TryGetAsync("k"); + + Assert.That(swapped, Is.False); + Assert.That(found, Is.True); + Assert.That(value.ToArray(), Is.EqualTo(actual.ToArray()), "value must be unchanged on a failed CAS"); + } + + [Test] + public async Task DeleteRemovesKeyAsync() + { + await using var store = new RaftSharedKeyValueStore(); + await store.SetAsync("k", ByteString.From(new byte[] { 1 })); + + bool deleted = await store.DeleteAsync("k"); + bool deletedAgain = await store.DeleteAsync("k"); + (bool found, _) = await store.TryGetAsync("k"); + + Assert.That(deleted, Is.True); + Assert.That(deletedAgain, Is.False); + Assert.That(found, Is.False); + } + + [Test] + public async Task ScanReturnsMatchingPrefixOnlyAsync() + { + await using var store = new RaftSharedKeyValueStore(); + await store.SetAsync("a/1", ByteString.From(new byte[] { 1 })); + await store.SetAsync("a/2", ByteString.From(new byte[] { 2 })); + await store.SetAsync("b/1", ByteString.From(new byte[] { 3 })); + + var keys = new List(); + await foreach (KeyValuePair entry in store.ScanAsync("a/")) + { + keys.Add(entry.Key); + } + + Assert.That(keys, Is.EquivalentTo(new[] { "a/1", "a/2" })); + } + + [Test] + public async Task WatchObservesSetAndDeleteForPrefixAsync() + { + await using var store = new RaftSharedKeyValueStore(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Pre-start the store so the watcher registers synchronously on the + // first MoveNextAsync, before any mutation is proposed. + await store.TryGetAsync("warmup", cts.Token); + + await using IAsyncEnumerator enumerator = + store.WatchAsync("a/", cts.Token).GetAsyncEnumerator(); + + ValueTask first = enumerator.MoveNextAsync(); + await store.SetAsync("b/ignored", ByteString.From(new byte[] { 0 }), cts.Token); + await store.SetAsync("a/1", ByteString.From(new byte[] { 1 }), cts.Token); + + Assert.That(await first, Is.True); + Assert.That(enumerator.Current.Kind, Is.EqualTo(KeyValueChangeKind.Set)); + Assert.That(enumerator.Current.Key, Is.EqualTo("a/1")); + + ValueTask second = enumerator.MoveNextAsync(); + await store.DeleteAsync("a/1", cts.Token); + + Assert.That(await second, Is.True); + Assert.That(enumerator.Current.Kind, Is.EqualTo(KeyValueChangeKind.Delete)); + Assert.That(enumerator.Current.Key, Is.EqualTo("a/1")); + } + + [Test] + public async Task ConcurrentCompareAndSwapHasExactlyOneWinnerAsync() + { + await using var store = new RaftSharedKeyValueStore(); + const int contenders = 24; + + // Every contender races to create the same key from absent. Because + // the consensus log is a single total order, exactly one wins. + IEnumerable> races = Enumerable.Range(0, contenders).Select(ii => + store.CompareAndSwapAsync("leader", default, ByteString.From(new[] { (byte)ii })).AsTask()); + bool[] results = await Task.WhenAll(races); + + Assert.That(results.Count(won => won), Is.EqualTo(1), "exactly one compare-and-swap may win"); + } + + [Test] + public async Task TwoReplicasConvergeOnSharedClusterAsync() + { + var cluster = new InProcessRaftCluster(); + await using InProcessRaftConsensus consensus1 = cluster.CreateNode(1); + await using InProcessRaftConsensus consensus2 = cluster.CreateNode(2); + await using var store1 = new RaftSharedKeyValueStore(consensus1); + await using var store2 = new RaftSharedKeyValueStore(consensus2); + + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + + // Start both replicas (registers each consensus node with the + // cluster) before writing, so the broadcast reaches both. + await store1.TryGetAsync("warmup", cts.Token); + await store2.TryGetAsync("warmup", cts.Token); + + ByteString payload = ByteString.From(new byte[] { 42 }); + await store1.SetAsync("shared", payload, cts.Token); + + ByteString observed = await WaitForValueAsync(store2, "shared", cts.Token); + Assert.That(observed.ToArray(), Is.EqualTo(payload.ToArray())); + } + + [Test] + public void ProposalTimesOutWhenNoCommitOccurs() + { + // Regression: a proposal must never hang when there is no leader / + // quorum to commit it; the commit timeout fails it instead. + Assert.That(async () => + { + await using var consensus = new NeverCommitsConsensus(); + await using var store = new RaftSharedKeyValueStore( + consensus, ownsConsensus: false, commitTimeout: TimeSpan.FromMilliseconds(200)); + await store.SetAsync("k", ByteString.From(new byte[] { 1 })); + }, Throws.TypeOf()); + } + + private static async Task WaitForValueAsync( + RaftSharedKeyValueStore store, + string key, + CancellationToken ct) + { + while (true) + { + (bool found, ByteString value) = await store.TryGetAsync(key, ct); + if (found) + { + return value; + } + await Task.Delay(10, ct); + } + } + + /// + /// A consensus replica that accepts proposals but never commits them (it never yields on + /// ), modelling a no-leader / lost-quorum window. + /// + private sealed class NeverCommitsConsensus : IRaftConsensus + { + public bool IsLeader => true; + + public event Action LeadershipChanged { add { } remove { } } + + public ChannelReader> Committed => m_committed.Reader; + + public ValueTask StartAsync(CancellationToken ct = default) => default; + + public ValueTask ProposeAsync(ReadOnlyMemory command, CancellationToken ct = default) => default; + + public ValueTask CampaignAsync(CancellationToken ct = default) => default; + + public ValueTask DisposeAsync() + { + m_committed.Writer.TryComplete(); + return default; + } + + private readonly Channel> m_committed = + Channel.CreateUnbounded>(); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RedundancyConsistencyBuilderExtensionsTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RedundancyConsistencyBuilderExtensionsTests.cs new file mode 100644 index 0000000000..e8313b0555 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Raft/RedundancyConsistencyBuilderExtensionsTests.cs @@ -0,0 +1,124 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for : consistency-mode DI selection. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class RedundancyConsistencyBuilderExtensionsTests + { + [Test] + public async Task StrongModeRegistersRaftStoreAndElectionAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency(RedundancyConsistencyMode.Strong); + await using ServiceProvider provider = services.BuildServiceProvider(); + + ISharedKeyValueStore store = provider.GetRequiredService(); + ILeaderElection election = provider.GetRequiredService(); + + Assert.That(store, Is.InstanceOf()); + Assert.That(election, Is.InstanceOf()); + + bool created = await store.CompareAndSwapAsync("k", default, ByteString.From(new byte[] { 1 })); + Assert.That(created, Is.True, "the strong store provides a linearizable compare-and-swap"); + Assert.That(election.IsLeader, Is.True, "the single-node default replica is the leader once used"); + } + + [Test] + public async Task EventualModeRegistersHybridStoreAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency(); + await using ServiceProvider provider = services.BuildServiceProvider(); + + ISharedKeyValueStore store = provider.GetRequiredService(); + Assert.That(store, Is.InstanceOf()); + + // Strong-prefix keys get linearizable CAS; bulk keys are stored too. + bool created = await store.CompareAndSwapAsync("nonce/x", default, ByteString.From(new byte[] { 1 })); + await store.SetAsync("node/1", ByteString.From(new byte[] { 2 })); + (bool found, _) = await store.TryGetAsync("node/1"); + + Assert.That(created, Is.True); + Assert.That(found, Is.True); + } + + [Test] + public async Task UseRaftLeaderElectionFalseSkipsElectionRegistrationAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency(options => + { + options.Mode = RedundancyConsistencyMode.Strong; + options.UseRaftLeaderElection = false; + }); + await using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That(provider.GetService(), Is.Null); + Assert.That(provider.GetRequiredService(), + Is.InstanceOf()); + } + + [Test] + public async Task ComposesBeforeUseDistributedAddressSpaceAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency(RedundancyConsistencyMode.Strong) + .UseDistributedAddressSpace(); + await using ServiceProvider provider = services.BuildServiceProvider(); + + // UseDistributedAddressSpace uses TryAddSingleton, so the strong + // consistency registration wins. + Assert.That(provider.GetRequiredService(), + Is.InstanceOf()); + Assert.That(provider.GetRequiredService(), + Is.InstanceOf()); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/RecordProtectionGuardTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/RecordProtectionGuardTests.cs new file mode 100644 index 0000000000..7bc00660d9 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/RecordProtectionGuardTests.cs @@ -0,0 +1,106 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using Microsoft.Extensions.DependencyInjection; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Tests for : mirrored state must fail + /// closed (refuse to start) when an external shared store is configured + /// without an . + /// + [TestFixture] + [Category("Distributed")] + public sealed class RecordProtectionGuardTests + { + [Test] + public void InMemoryStoreWithoutProtectorReturnsNull() + { + using ServiceProvider services = new ServiceCollection() + .AddSingleton(new InMemorySharedKeyValueStore()) + .BuildServiceProvider(); + + Assert.That(RecordProtectionGuard.ResolveProtectorOrThrow(services), Is.Null); + } + + [Test] + public void ExternalStoreWithoutProtectorThrows() + { + using ServiceProvider services = new ServiceCollection() + .AddSingleton(Mock.Of()) + .BuildServiceProvider(); + + Assert.That( + () => RecordProtectionGuard.ResolveProtectorOrThrow(services), + Throws.InvalidOperationException); + } + + [Test] + public void ExternalStoreWithProtectorReturnsProtector() + { + IRecordProtector protector = Mock.Of(); + using ServiceProvider services = new ServiceCollection() + .AddSingleton(Mock.Of()) + .AddSingleton(protector) + .BuildServiceProvider(); + + Assert.That( + RecordProtectionGuard.ResolveProtectorOrThrow(services), + Is.SameAs(protector)); + } + + [Test] + public void ExternalStoreWithExplicitNullProtectorIsAllowed() + { + using ServiceProvider services = new ServiceCollection() + .AddSingleton(Mock.Of()) + .AddSingleton(NullRecordProtector.Instance) + .BuildServiceProvider(); + + Assert.That( + RecordProtectionGuard.ResolveProtectorOrThrow(services), + Is.SameAs(NullRecordProtector.Instance)); + } + + [Test] + public void NullServicesThrowsArgumentNull() + { + Assert.That( + () => RecordProtectionGuard.ResolveProtectorOrThrow(null!), + Throws.ArgumentNullException); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/BandedServerDirectionPolicyTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/BandedServerDirectionPolicyTests.cs new file mode 100644 index 0000000000..dcc8da3e74 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/BandedServerDirectionPolicyTests.cs @@ -0,0 +1,188 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class BandedServerDirectionPolicyTests + { + [Test] + public async Task ServesSelfWhenLocalIsLeastLoadedInTopTierAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([Peer("B", 255, 80)]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 10); + + Assert.That(target, Is.Null, "the local Server is least-loaded among equally-healthy peers"); + } + + [Test] + public async Task RedirectsToLeastLoadedPeerInSameTierAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([Peer("B", 255, 10)]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 80); + + Assert.That(target, Is.EqualTo("B")); + } + + [Test] + public async Task RedirectsToHealthierPeerRegardlessOfLoadAsync() + { + // Active/passive: local standby is NoData, the active peer is Healthy. + var policy = new BandedServerDirectionPolicy( + new FakeView([Peer("B", 255, 200)]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", ServiceLevels.NoData, 0); + + Assert.That(target, Is.EqualTo("B"), "a strictly-healthier peer wins even when heavily loaded"); + } + + [Test] + public async Task ServesSelfWhenLocalIsStrictlyHealthiestAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([Peer("B", 100, 0)]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 90); + + Assert.That(target, Is.Null, "no peer is in a higher health tier than the healthy local Server"); + } + + [Test] + public async Task IgnoresLowerTierPeersForLoadBalancingAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([Peer("B", 255, 10), Peer("C", 100, 0)]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 90); + + Assert.That(target, Is.EqualTo("B"), "only the top health tier participates; the degraded peer is ignored"); + } + + [Test] + public async Task RandomTieBreakSelectsWithinEqualLoadBandAsync() + { + ArrayOf peers = [Peer("B", 255, 0), Peer("C", 255, 5)]; + + var pickFirst = new BandedServerDirectionPolicy(new FakeView(peers), new LoadDirectionOptions(), _ => 0); + var pickSecond = new BandedServerDirectionPolicy(new FakeView(peers), new LoadDirectionOptions(), _ => 1); + + string? first = await pickFirst.SelectTargetServerUriAsync("A", 255, 100); + string? second = await pickSecond.SelectTargetServerUriAsync("A", 255, 100); + + Assert.That(first, Is.EqualTo("B")); + Assert.That(second, Is.EqualTo("C"), "B and C share the lowest load band; the selector breaks the tie"); + } + + [Test] + public async Task PeerWithUnknownLoadIsDeprioritizedAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([Peer("B", 255, load: null)]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 50); + + Assert.That(target, Is.Null, "a peer with an unknown load must not be preferred over the known local load"); + } + + [Test] + public async Task EmptyViewServesSelfAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([]), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 0); + + Assert.That(target, Is.Null); + } + + [Test] + public async Task ViewFailureServesSelfAsync() + { + var policy = new BandedServerDirectionPolicy( + new FakeView([], throwOnRead: true), new LoadDirectionOptions(), _ => 0); + + string? target = await policy.SelectTargetServerUriAsync("A", 255, 0); + + Assert.That(target, Is.Null, "a stale/unreadable peer view must fail safe to the local Server"); + } + + private static PeerDirectionRecord Peer(string uri, byte level, byte? load = null) + { + return new PeerDirectionRecord + { + ServerUri = uri, + ServiceLevel = level, + LoadWeight = load ?? 0, + LoadKnown = load.HasValue + }; + } + + private sealed class FakeView : IPeerDirectionView + { + public FakeView(ArrayOf peers, bool throwOnRead = false) + { + m_peers = peers; + m_throw = throwOnRead; + } + + public ValueTask> GetPeersAsync(CancellationToken cancellationToken = default) + { + if (m_throw) + { + throw new InvalidOperationException("simulated store failure"); + } + return new ValueTask>(m_peers); + } + + private readonly ArrayOf m_peers; + private readonly bool m_throw; + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LeaderElectionTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LeaderElectionTests.cs new file mode 100644 index 0000000000..fdb45da7cb --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LeaderElectionTests.cs @@ -0,0 +1,159 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the leader-election implementations. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class LeaderElectionTests + { + private static readonly TimeSpan LeaseDuration = TimeSpan.FromSeconds(30); + private static readonly TimeSpan RenewInterval = TimeSpan.FromSeconds(10); + + [Test] + public async Task StaticLeaderElectionReportsFixedRoleAsync() + { + await using var leader = new StaticLeaderElection(true); + await using var follower = new StaticLeaderElection(false); + + Assert.That(leader.IsLeader, Is.True); + Assert.That(follower.IsLeader, Is.False); + } + + [Test] + public async Task FirstAcquirerBecomesLeaderAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + await using var election = NewElection(kv, "A", time); + + bool acquired = await election.TryAcquireOrRenewAsync(); + + Assert.That(acquired, Is.True); + Assert.That(election.IsLeader, Is.True); + } + + [Test] + public async Task SecondReplicaIsFollowerWhileLeaseHeldAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + await using var a = NewElection(kv, "A", time); + await using var b = NewElection(kv, "B", time); + + Assert.That(await a.TryAcquireOrRenewAsync(), Is.True); + Assert.That(await b.TryAcquireOrRenewAsync(), Is.False); + Assert.That(b.IsLeader, Is.False); + } + + [Test] + public async Task LeaderRenewsWithoutLosingLeadershipAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + await using var a = NewElection(kv, "A", time); + await using var b = NewElection(kv, "B", time); + + Assert.That(await a.TryAcquireOrRenewAsync(), Is.True); + time.Advance(TimeSpan.FromSeconds(10)); + Assert.That(await a.TryAcquireOrRenewAsync(), Is.True, "leader should renew"); + Assert.That(await b.TryAcquireOrRenewAsync(), Is.False, "standby cannot take a live lease"); + } + + [Test] + public async Task StandbyTakesOverAfterLeaseExpiresAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + await using var a = NewElection(kv, "A", time); + await using var b = NewElection(kv, "B", time); + + Assert.That(await a.TryAcquireOrRenewAsync(), Is.True); + time.Advance(LeaseDuration + TimeSpan.FromSeconds(1)); + + Assert.That(await b.TryAcquireOrRenewAsync(), Is.True, "standby takes over the expired lease"); + Assert.That(await a.TryAcquireOrRenewAsync(), Is.False, "old leader becomes follower"); + Assert.That(b.IsLeader, Is.True); + Assert.That(a.IsLeader, Is.False); + } + + [Test] + public async Task ReleaseOnDisposeAllowsImmediateTakeoverAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + + SharedStoreLeaseElection a = NewElection(kv, "A", time); + Assert.That(await a.TryAcquireOrRenewAsync(), Is.True); + await a.DisposeAsync(); + + await using var b = NewElection(kv, "B", time); + Assert.That(await b.TryAcquireOrRenewAsync(), Is.True, "released lease can be taken immediately"); + } + + [Test] + public async Task LeadershipChangedFiresOnAcquireAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + await using var a = NewElection(kv, "A", time); + bool? observed = null; + a.LeadershipChanged += value => observed = value; + + await a.TryAcquireOrRenewAsync(); + + Assert.That(observed, Is.True); + } + + private static SharedStoreLeaseElection NewElection( + ISharedKeyValueStore store, + string nodeId, + TimeProvider time) + { + return new SharedStoreLeaseElection(store, "lease/asp", nodeId, LeaseDuration, RenewInterval, time); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LoadDirectionServerIntegrationTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LoadDirectionServerIntegrationTests.cs new file mode 100644 index 0000000000..ad1aeb09f8 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LoadDirectionServerIntegrationTests.cs @@ -0,0 +1,161 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.TestFramework; +using Quickstarts.ReferenceServer; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Integration test that drives a real, fully-started whose + /// is a configured , and verifies + /// end to end that a GetEndpoints request on the balancing URL is answered with a peer's endpoints while a + /// normal request serves the local server. Closes the seam-wiring gap the director unit tests cannot exercise. + /// + [TestFixture] + [Category("Distributed")] + [NonParallelizable] + public class LoadDirectionServerIntegrationTests + { + private const string BalancingUrl = "opc.tcp://balance.invalid:4840"; + + private InMemorySharedKeyValueStore m_kv = null!; + private LoadDirectionOptions m_options = null!; + private ServerLoadDirector m_director = null!; + private ServerFixture m_fixture = null!; + private StandardServer m_server = null!; + + [OneTimeSetUp] + public async Task OneTimeSetUpAsync() + { + m_kv = new InMemorySharedKeyValueStore(); + m_options = new LoadDirectionOptions { BalancingEndpointUrl = BalancingUrl }; + m_director = new ServerLoadDirector( + new ConstantServiceLevelProvider(100), // local is a Degraded standby + new ConstantLoadWeightProvider(200), + m_options); + + m_fixture = new ServerFixture(t => + { + var server = new ReferenceServer(t); + server.GetEndpointsDirector = m_director; + return server; + }); + m_server = await m_fixture.StartAsync(); + + IServiceMessageContext context = m_server.CurrentInstance.MessageContext; + string localServerUri = m_server.CurrentInstance.ServerUris.ToArray()[0]; + + var view = new SharedPeerDirectionView( + m_kv, context, NullRecordProtector.Instance, m_options, TimeProvider.System); + var policy = new BandedServerDirectionPolicy(view, m_options, _ => 0); + var directory = new SharedPeerEndpointDirectory(m_kv, context, NullRecordProtector.Instance, m_options); + var publisher = new SharedPeerEndpointPublisher( + m_kv, context, NullRecordProtector.Instance, m_options, localServerUri); + m_director.Configure(policy, directory, publisher, localServerUri); + + // Seed peer B: healthy, unloaded, with endpoints. + var bDirection = new SharedPeerDirectionPublisher( + m_kv, context, NullRecordProtector.Instance, m_options, TimeProvider.System, "urn:B"); + await bDirection.PublishServiceLevelAsync(255); + await bDirection.PublishLoadWeightAsync(0); + var bEndpoints = new SharedPeerEndpointPublisher( + m_kv, context, NullRecordProtector.Instance, m_options, "urn:B"); + await bEndpoints.PublishAsync([MakeEndpoint("opc.tcp://b.example:4840", "urn:B")]); + } + + [OneTimeTearDown] + public async Task OneTimeTearDownAsync() + { + if (m_fixture != null) + { + await m_fixture.StopAsync(); + } + m_kv?.Dispose(); + } + + [Test] + public async Task GetEndpointsOnBalancingUrlRedirectsToHealthierPeerAsync() + { + GetEndpointsResponse response = await m_server.GetEndpointsAsync( + CreateChannelContext(), new RequestHeader(), BalancingUrl, default, default, RequestLifetime.None); + + EndpointDescription[] endpoints = response.Endpoints.ToArray(); + Assert.That(endpoints, Has.Length.EqualTo(1)); + Assert.That(endpoints[0].Server?.ApplicationUri, Is.EqualTo("urn:B")); + } + + [Test] + public async Task GetEndpointsOnNormalUrlServesLocalServerAsync() + { + string normalUrl = m_server.GetEndpoints().ToArray()[0].EndpointUrl; + + GetEndpointsResponse response = await m_server.GetEndpointsAsync( + CreateChannelContext(), new RequestHeader(), normalUrl, default, default, RequestLifetime.None); + + EndpointDescription[] endpoints = response.Endpoints.ToArray(); + Assert.That(endpoints, Is.Not.Empty, "a normal request returns the local server's own endpoints"); + Assert.That( + endpoints.Any(e => string.Equals(e.Server?.ApplicationUri, "urn:B", StringComparison.Ordinal)), + Is.False, + "a normal request must not be redirected to a peer"); + } + + private SecureChannelContext CreateChannelContext() + { + EndpointDescription endpoint = m_server.GetEndpoints().ToArray()[0]; + return new SecureChannelContext("loaddir-test", endpoint, RequestEncoding.Binary, null, null, null); + } + + private static EndpointDescription MakeEndpoint(string url, string serverUri) + { + return new EndpointDescription + { + EndpointUrl = url, + SecurityMode = MessageSecurityMode.None, + SecurityPolicyUri = SecurityPolicies.None, + Server = new ApplicationDescription + { + ApplicationUri = serverUri, + ApplicationType = ApplicationType.Server + } + }; + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LoadDirectionStrongRoutingTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LoadDirectionStrongRoutingTests.cs new file mode 100644 index 0000000000..3cd1e13e24 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/LoadDirectionStrongRoutingTests.cs @@ -0,0 +1,89 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the load-direction StrongEligibility composition with the hybrid consistency store. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class LoadDirectionStrongRoutingTests + { + [Test] + public async Task StrongEligibilityRoutesEligibilityKeyspacesToRaftAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency() + .UseServerLoadDirection(options => + { + options.BalancingEndpointUrl = "opc.tcp://ha:4840"; + options.StrongEligibility = true; + }); + await using ServiceProvider provider = services.BuildServiceProvider(); + + var store = (HybridSharedKeyValueStore)provider.GetRequiredService(); + + Assert.That(store.IsStrongKey("svc/urn:a"), Is.True, "the health eligibility keyspace is linearizable"); + Assert.That(store.IsStrongKey("endpoint/urn:a"), Is.True, "the endpoint directory is linearizable"); + Assert.That(store.IsStrongKey("load/urn:a"), Is.False, "the high-churn load weight stays eventual"); + Assert.That(store.IsStrongKey("nonce/x"), Is.True, "the default strong keyspaces are preserved"); + Assert.That(store.IsStrongKey("node/1"), Is.False, "bulk keys stay eventual"); + } + + [Test] + public async Task DefaultLeavesEligibilityKeyspacesEventualAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa().AddServer(_ => { }) + .UseRedundancyConsistency() + .UseServerLoadDirection(options => options.BalancingEndpointUrl = "opc.tcp://ha:4840"); + await using ServiceProvider provider = services.BuildServiceProvider(); + + var store = (HybridSharedKeyValueStore)provider.GetRequiredService(); + + Assert.That(store.IsStrongKey("svc/urn:a"), Is.False, "eligibility stays eventual by default"); + Assert.That(store.IsStrongKey("endpoint/urn:a"), Is.False); + Assert.That(store.IsStrongKey("nonce/x"), Is.True, "the default strong keyspaces are preserved"); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/PeerDirectionViewTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/PeerDirectionViewTests.cs new file mode 100644 index 0000000000..43ccc34741 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/PeerDirectionViewTests.cs @@ -0,0 +1,191 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the GetEndpoints load-direction peer signal publish/view + /// ( / ). + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class PeerDirectionViewTests + { + [Test] + public async Task PublishAndViewRoundTripReturnsFreshPeerAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + var time = new FakeTimeProvider(); + + var publisher = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:server:a"); + await publisher.PublishServiceLevelAsync(200); + await publisher.PublishLoadWeightAsync(30); + + var view = new SharedPeerDirectionView( + store, context, NullRecordProtector.Instance, options, time); + PeerDirectionRecord[] peers = (await view.GetPeersAsync()).ToArray(); + + Assert.That(peers, Has.Length.EqualTo(1)); + Assert.That(peers[0].ServerUri, Is.EqualTo("urn:server:a")); + Assert.That(peers[0].ServiceLevel, Is.EqualTo((byte)200)); + Assert.That(peers[0].LoadKnown, Is.True); + Assert.That(peers[0].LoadWeight, Is.EqualTo((byte)30)); + } + + [Test] + public async Task ViewExcludesStaleHealthRecordAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions { StalenessWindow = TimeSpan.FromSeconds(15) }; + var time = new FakeTimeProvider(); + + var publisher = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:server:a"); + await publisher.PublishServiceLevelAsync(200); + await publisher.PublishLoadWeightAsync(30); + + time.Advance(TimeSpan.FromSeconds(20)); + + var view = new SharedPeerDirectionView( + store, context, NullRecordProtector.Instance, options, time); + PeerDirectionRecord[] peers = (await view.GetPeersAsync()).ToArray(); + + Assert.That(peers, Is.Empty, "a peer whose health record is stale must be aged out"); + } + + [Test] + public async Task ViewReportsLoadUnknownWhenLoadStaleButHealthFreshAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions { StalenessWindow = TimeSpan.FromSeconds(15) }; + var time = new FakeTimeProvider(); + + var publisher = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:server:a"); + await publisher.PublishLoadWeightAsync(30); + time.Advance(TimeSpan.FromSeconds(20)); + await publisher.PublishServiceLevelAsync(200); + + var view = new SharedPeerDirectionView( + store, context, NullRecordProtector.Instance, options, time); + PeerDirectionRecord[] peers = (await view.GetPeersAsync()).ToArray(); + + Assert.That(peers, Has.Length.EqualTo(1)); + Assert.That(peers[0].ServiceLevel, Is.EqualTo((byte)200)); + Assert.That(peers[0].LoadKnown, Is.False, "a stale load record must be reported as unknown"); + Assert.That(peers[0].LoadWeight, Is.Zero); + } + + [Test] + public async Task ViewDropsMalformedHealthRecordAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + var time = new FakeTimeProvider(); + + await store.SetAsync("svc/urn:server:x", ByteString.From(new byte[] { 9, 9, 9 })); + + var view = new SharedPeerDirectionView( + store, context, NullRecordProtector.Instance, options, time); + PeerDirectionRecord[] peers = (await view.GetPeersAsync()).ToArray(); + + Assert.That(peers, Is.Empty, "an undecodable record must be dropped (fail-closed)"); + } + + [Test] + public async Task ViewReturnsAllHealthyPeersAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + var time = new FakeTimeProvider(); + + var a = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:server:a"); + var b = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:server:b"); + await a.PublishServiceLevelAsync(200); + await b.PublishServiceLevelAsync(210); + + var view = new SharedPeerDirectionView( + store, context, NullRecordProtector.Instance, options, time); + PeerDirectionRecord[] peers = (await view.GetPeersAsync()).ToArray(); + + Assert.That(peers.Select(p => p.ServerUri), Is.EquivalentTo(new[] { "urn:server:a", "urn:server:b" })); + Assert.That(peers.Single(p => p.ServerUri == "urn:server:b").ServiceLevel, Is.EqualTo((byte)210)); + } + + [Test] + public async Task RepublishOverwritesPreviousSignalAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + var time = new FakeTimeProvider(); + + var publisher = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:server:a"); + await publisher.PublishServiceLevelAsync(200); + time.Advance(TimeSpan.FromSeconds(1)); + await publisher.PublishServiceLevelAsync(120); + + var view = new SharedPeerDirectionView( + store, context, NullRecordProtector.Instance, options, time); + PeerDirectionRecord[] peers = (await view.GetPeersAsync()).ToArray(); + + Assert.That(peers, Has.Length.EqualTo(1)); + Assert.That(peers[0].ServiceLevel, Is.EqualTo((byte)120), "the latest published health value must win"); + } + + private static ServiceMessageContext CreateContext() + { + return ServiceMessageContext.CreateEmpty(NUnitTelemetryContext.Create()); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/PeerEndpointDirectoryTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/PeerEndpointDirectoryTests.cs new file mode 100644 index 0000000000..4d5bd50902 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/PeerEndpointDirectoryTests.cs @@ -0,0 +1,141 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the GetEndpoints load-direction peer endpoint directory + /// ( / ). + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class PeerEndpointDirectoryTests + { + [Test] + public async Task PublishAndReadRoundTripReturnsEndpointsAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + + var publisher = new SharedPeerEndpointPublisher( + store, context, NullRecordProtector.Instance, options, "urn:server:a"); + ArrayOf published = + [ + Endpoint("opc.tcp://a:4840", MessageSecurityMode.None, "urn:server:a"), + Endpoint("opc.tcp://a:4841", MessageSecurityMode.SignAndEncrypt, "urn:server:a") + ]; + await publisher.PublishAsync(published); + + var directory = new SharedPeerEndpointDirectory( + store, context, NullRecordProtector.Instance, options); + EndpointDescription[] resolved = (await directory.GetEndpointsAsync("urn:server:a")).ToArray(); + + Assert.That(resolved, Has.Length.EqualTo(2)); + Assert.That(resolved[0].EndpointUrl, Is.EqualTo("opc.tcp://a:4840")); + Assert.That(resolved[1].EndpointUrl, Is.EqualTo("opc.tcp://a:4841")); + Assert.That(resolved[1].SecurityMode, Is.EqualTo(MessageSecurityMode.SignAndEncrypt)); + } + + [Test] + public async Task UnknownPeerReturnsEmptyAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + + var directory = new SharedPeerEndpointDirectory( + store, context, NullRecordProtector.Instance, options); + EndpointDescription[] resolved = (await directory.GetEndpointsAsync("urn:server:missing")).ToArray(); + + Assert.That(resolved, Is.Empty); + } + + [Test] + public async Task EmptyServerUriReturnsEmptyAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + + var directory = new SharedPeerEndpointDirectory( + store, context, NullRecordProtector.Instance, options); + EndpointDescription[] resolved = (await directory.GetEndpointsAsync(string.Empty)).ToArray(); + + Assert.That(resolved, Is.Empty); + } + + [Test] + public async Task MalformedRecordReturnsEmptyAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions(); + await store.SetAsync("endpoint/urn:server:a", ByteString.From(new byte[] { 7, 7, 7 })); + + var directory = new SharedPeerEndpointDirectory( + store, context, NullRecordProtector.Instance, options); + EndpointDescription[] resolved = (await directory.GetEndpointsAsync("urn:server:a")).ToArray(); + + Assert.That(resolved, Is.Empty, "an undecodable endpoint record must be dropped (fail-closed)"); + } + + private static EndpointDescription Endpoint(string url, MessageSecurityMode mode, string serverUri) + { + return new EndpointDescription + { + EndpointUrl = url, + SecurityMode = mode, + SecurityPolicyUri = SecurityPolicies.None, + Server = new ApplicationDescription + { + ApplicationUri = serverUri, + ApplicationType = ApplicationType.Server + } + }; + } + + private static ServiceMessageContext CreateContext() + { + return ServiceMessageContext.CreateEmpty(NUnitTelemetryContext.Create()); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/RedundancyDiscoveryTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/RedundancyDiscoveryTests.cs new file mode 100644 index 0000000000..3b391a78b3 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/RedundancyDiscoveryTests.cs @@ -0,0 +1,233 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Moq; +using NUnit.Framework; +using Opc.Ua.Configuration; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Tests for redundant server-set discovery integration. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class RedundancyDiscoveryTests + { + [Test] + public void AddServerRedundancyAddsNtrsCapabilityForNonTransparentMode() + { + OpcUaServerOptions options = CreateOptionsWithRedundancy(RedundancySupport.Hot); + var configurationBuilder = new Mock(); + configurationBuilder + .Setup(builder => builder.AddServerCapabilities("NTRS")) + .Returns(configurationBuilder.Object); + + options.ConfigureBuilder!(configurationBuilder.Object); + + configurationBuilder.Verify(builder => builder.AddServerCapabilities("NTRS"), Times.Once); + } + + [Test] + public void AddServerRedundancyDoesNotAddNtrsCapabilityForTransparentOrNone() + { + foreach (RedundancySupport mode in new[] { RedundancySupport.None, RedundancySupport.Transparent }) + { + OpcUaServerOptions options = CreateOptionsWithRedundancy(mode); + var configurationBuilder = new Mock(); + + options.ConfigureBuilder!(configurationBuilder.Object); + + configurationBuilder.Verify( + builder => builder.AddServerCapabilities("NTRS"), + Times.Never); + } + } + + [Test] + public async Task FindServersReturnsRedundantPeerDescriptionsAsync() + { + using var server = new TestStandardServer(); + server.RedundantServerSetProvider = CreateProvider(); + + FindServersResponse response = await server.FindServersAsync( + null!, + new RequestHeader(), + null, + [], + [], + RequestLifetime.None); + + ApplicationDescription[] servers = response.Servers.Memory.ToArray(); + Assert.That(servers.Select(description => description.ApplicationUri), + Is.EqualTo(new[] { "urn:local", "urn:peer-a", "urn:peer-b" })); + ApplicationDescription peer = servers.Single( + description => description.ApplicationUri == "urn:peer-a"); + Assert.That(peer.DiscoveryUrls, Is.EqualTo(new[] { "opc.tcp://peer-a:4840" })); + } + + [Test] + public async Task FindServersWithoutProviderReturnsOnlyLocalDescriptionAsync() + { + using var server = new TestStandardServer(); + + FindServersResponse response = await server.FindServersAsync( + null!, + new RequestHeader(), + null, + [], + [], + RequestLifetime.None); + + Assert.That(response.Servers.Memory.ToArray().Select(description => description.ApplicationUri), + Is.EqualTo(new[] { "urn:local" })); + } + + [Test] + public async Task FindServersFiltersRedundantPeerDescriptionsByServerUriAsync() + { + using var server = new TestStandardServer(); + server.RedundantServerSetProvider = CreateProvider(); + + FindServersResponse response = await server.FindServersAsync( + null!, + new RequestHeader(), + null, + [], + ["urn:peer-b"], + RequestLifetime.None); + + Assert.That(response.Servers.Memory.ToArray().Select(description => description.ApplicationUri), + Is.EqualTo(new[] { "urn:peer-b" })); + } + + private static OpcUaServerOptions CreateOptionsWithRedundancy(RedundancySupport mode) + { + var services = new ServiceCollection(); + var builder = new TestServerBuilder(services); + builder.AddServerRedundancy(options => options.Mode = mode); + using ServiceProvider provider = services.BuildServiceProvider(); + return provider.GetRequiredService>().Value; + } + + private static ConfiguredRedundantServerSetProvider CreateProvider() + { + var options = new ServerRedundancyOptions + { + Mode = RedundancySupport.Hot + }; + options.AddRedundantPeer("urn:peer-a", ["opc.tcp://peer-a:4840"]); + options.AddRedundantPeer("urn:peer-b", ["opc.tcp://peer-b:4840"]); + return new ConfiguredRedundantServerSetProvider(options); + } + + private sealed class TestServerBuilder : IOpcUaServerBuilder + { + public TestServerBuilder(IServiceCollection services) + { + Services = services; + } + + public IServiceCollection Services { get; } + + public IOpcUaServerBuilder AddNodeManager() + where TFactory : class, IAsyncNodeManagerFactory + { + return this; + } + + public IOpcUaServerBuilder AddSyncNodeManager() + where TFactory : class, INodeManagerFactory + { + return this; + } + } + + private sealed class TestStandardServer : StandardServer + { + public TestStandardServer() + : base(NUnitTelemetryContext.Create()) + { + BaseAddresses = + [ + new BaseAddress + { + Url = new Uri("opc.tcp://localhost:4840"), + DiscoveryUrl = new Uri("opc.tcp://localhost:4840") + } + ]; + m_endpoints = + [ + new EndpointDescription + { + EndpointUrl = "opc.tcp://localhost:4840", + Server = new ApplicationDescription + { + ApplicationUri = "urn:local", + ApplicationName = new LocalizedText("Local"), + ApplicationType = ApplicationType.Server, + DiscoveryUrls = ["opc.tcp://localhost:4840"] + } + } + ]; + } + + public override ArrayOf GetEndpoints() + { + return m_endpoints; + } + + protected override void ValidateRequest([NotNull] RequestHeader? requestHeader) + { + if (requestHeader == null) + { + throw new ServiceResultException(StatusCodes.BadRequestHeaderInvalid); + } + } + + private readonly ArrayOf m_endpoints; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/RequestServerStateChangeStartupTaskTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/RequestServerStateChangeStartupTaskTests.cs new file mode 100644 index 0000000000..e1be812497 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/RequestServerStateChangeStartupTaskTests.cs @@ -0,0 +1,380 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class RequestServerStateChangeStartupTaskTests + { + [Test] + public void ConstructorThrowsOnNullOptions() + { + Assert.That( + () => new RequestServerStateChangeStartupTask((RequestServerStateChangeOptions)null!), + Throws.ArgumentNullException); + } + + [Test] + public void OnServerStartedThrowsOnNullServer() + { + var task = new RequestServerStateChangeStartupTask(); + + Assert.That(async () => await task.OnServerStartedAsync(null!), Throws.ArgumentNullException); + } + + [Test] + public async Task RequestShutdownPublishesMaintenanceAndEstimatedReturnTimeAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + Mock controller = new(); + RequestServerStateChangeStartupTask task = CreateTask(controller.Object); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + DateTimeUtc estimatedReturnTime = DateTimeUtc.Now; + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Shutdown, + estimatedReturnTime, + 10, + new LocalizedText("maintenance"), + restart: false); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + Assert.That(method.NodeId, Is.EqualTo(MethodIds.Server_RequestServerStateChange)); + controller.Verify(c => c.SetServiceLevel(ServiceLevels.Maintenance), Times.Once); + PropertyState estimatedReturnTimeNode = + loaded.Manager.FindPredefinedNode>( + VariableIds.Server_EstimatedReturnTime); + Assert.That(estimatedReturnTimeNode.Value, Is.EqualTo(estimatedReturnTime)); + } + + [Test] + public async Task RequestFailedPublishesNoDataWhenNoControllerIsConfiguredAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + RequestServerStateChangeStartupTask task = CreateTask(); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Failed, + DateTimeUtc.Now, + 0, + new LocalizedText("failed"), + restart: false); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + PropertyState serviceLevel = + loaded.Manager.FindPredefinedNode>(VariableIds.Server_ServiceLevel); + Assert.That(serviceLevel.Value, Is.EqualTo(ServiceLevels.NoData)); + } + + [Test] + public async Task RequestReturnsAccessDeniedWhenAdminValidationFailsAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + var options = new RequestServerStateChangeOptions + { + AdminAccessValidator = _ => throw new ServiceResultException(StatusCodes.BadUserAccessDenied) + }; + var task = new RequestServerStateChangeStartupTask(options); + PropertyState serviceLevel = + loaded.Manager.FindPredefinedNode>(VariableIds.Server_ServiceLevel); + serviceLevel.Value = ServiceLevels.Maximum; + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Shutdown, + DateTimeUtc.Now, + 0, + LocalizedText.Null, + restart: false); + + Assert.That(result.StatusCode.Code, Is.EqualTo(StatusCodes.BadUserAccessDenied)); + Assert.That(serviceLevel.Value, Is.EqualTo(ServiceLevels.Maximum)); + } + + [Test] + public async Task RequestSuspendedPublishesMaintenanceAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + RequestServerStateChangeStartupTask task = CreateTask(); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Suspended, + DateTimeUtc.Now, + 0, + LocalizedText.Null, + restart: false); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + PropertyState serviceLevel = + loaded.Manager.FindPredefinedNode>(VariableIds.Server_ServiceLevel); + Assert.That(serviceLevel.Value, Is.EqualTo(ServiceLevels.Maintenance)); + } + + [Test] + public async Task RequestTestPublishesNoDataAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + RequestServerStateChangeStartupTask task = CreateTask(); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Test, + DateTimeUtc.Now, + 0, + LocalizedText.Null, + restart: false); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + PropertyState serviceLevel = + loaded.Manager.FindPredefinedNode>(VariableIds.Server_ServiceLevel); + Assert.That(serviceLevel.Value, Is.EqualTo(ServiceLevels.NoData)); + } + + [Test] + public async Task RequestUsesCustomServiceLevelSelectorAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + var options = new RequestServerStateChangeOptions + { + AdminAccessValidator = _ => { }, + ServiceLevelSelector = state => state == ServerState.Running ? (byte)200 : (byte)100 + }; + var task = new RequestServerStateChangeStartupTask(options); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Running, + DateTimeUtc.Now, + 0, + LocalizedText.Null, + restart: false); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + PropertyState serviceLevel = + loaded.Manager.FindPredefinedNode>(VariableIds.Server_ServiceLevel); + Assert.That(serviceLevel.Value, Is.EqualTo(200)); + } + + [Test] + public async Task RequestUpdatesServerStatusFieldsAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + RequestServerStateChangeStartupTask task = CreateTask(); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + DateTimeUtc estimatedReturnTime = new DateTimeUtc(638000000000000000); + LocalizedText reason = new LocalizedText("Planned maintenance"); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Shutdown, + estimatedReturnTime, + 300, + reason, + restart: false); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + ServerStatusState serverStatus = loaded.Manager.FindPredefinedNode( + VariableIds.Server_ServerStatus); + Assert.That(serverStatus.State!.Value, Is.EqualTo(ServerState.Shutdown)); + Assert.That(serverStatus.SecondsTillShutdown!.Value, Is.EqualTo(300)); + Assert.That(serverStatus.ShutdownReason!.Value.Text, Is.EqualTo("Planned maintenance")); + } + + [Test] + public async Task RequestAcceptsRestartFlagAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + RequestServerStateChangeStartupTask task = CreateTask(); + + await task.OnServerStartedAsync(loaded.Server.Object); + RequestServerStateChangeMethodState method = + loaded.Manager.FindPredefinedNode( + MethodIds.Server_RequestServerStateChange); + ServiceResult result = method.OnCall!( + loaded.Server.Object.DefaultSystemContext, + method, + ObjectIds.Server, + ServerState.Shutdown, + DateTimeUtc.Now, + 10, + LocalizedText.Null, + restart: true); + + Assert.That(result, Is.EqualTo(ServiceResult.Good)); + } + + private static RequestServerStateChangeStartupTask CreateTask( + IServiceLevelController? controller = null) + { + return new RequestServerStateChangeStartupTask( + new RequestServerStateChangeOptions + { + AdminAccessValidator = _ => { } + }, + controller); + } + + private static async Task CreateLoadedServerAsync() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + var namespaceUris = new NamespaceTable(); + namespaceUris.Append("http://opcfoundation.org/UA/"); + var serverUris = new StringTable(); + var typeTree = new TypeTable(namespaceUris); + var messageContext = new ServiceMessageContext(telemetry, EncodeableFactory.Create()) + { + NamespaceUris = namespaceUris, + ServerUris = serverUris + }; + + var server = new Mock(); + var coreNodeManager = new Mock(); + coreNodeManager.Setup(m => m.ImportNodesAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(new ValueTask()); + var masterNodeManager = new Mock(); + masterNodeManager.Setup(m => m.RemoveReferencesAsync( + It.IsAny>(), + It.IsAny())) + .Returns(new ValueTask()); + var configurationNodeManager = new Mock(); + + server.Setup(s => s.Telemetry).Returns(telemetry); + server.Setup(s => s.NamespaceUris).Returns(namespaceUris); + server.Setup(s => s.ServerUris).Returns(serverUris); + server.Setup(s => s.TypeTree).Returns(typeTree); + server.Setup(s => s.MessageContext).Returns(messageContext); + server.Setup(s => s.CoreNodeManager).Returns(coreNodeManager.Object); + server.Setup(s => s.NodeManager).Returns(masterNodeManager.Object); + server.Setup(s => s.Factory).Returns(EncodeableFactory.Create()); + server.Setup(s => s.ConfigurationNodeManager).Returns(configurationNodeManager.Object); + + var context = new ServerSystemContext(server.Object); + server.Setup(s => s.DefaultSystemContext).Returns(context); + + var configuration = new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration() + }; + var manager = new DiagnosticsNodeManager(server.Object, configuration, NullLogger.Instance); + await manager.CreateAddressSpaceAsync(new Dictionary>()); + ServerObjectState serverObject = manager.FindPredefinedNode(ObjectIds.Server); + server.Setup(s => s.ServerObject).Returns(serverObject); + server.Setup(s => s.DiagnosticsNodeManager).Returns(manager); + return new LoadedDiagnosticsServer(server, manager); + } + + private sealed class LoadedDiagnosticsServer : IDisposable + { + public LoadedDiagnosticsServer( + Mock server, + DiagnosticsNodeManager manager) + { + Server = server; + Manager = manager; + } + + public Mock Server { get; } + + public DiagnosticsNodeManager Manager { get; } + + public void Dispose() + { + Manager.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServerLoadDirectorTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServerLoadDirectorTests.cs new file mode 100644 index 0000000000..10d51ea88b --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServerLoadDirectorTests.cs @@ -0,0 +1,219 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Opc.Ua.Redundancy; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Integration tests for using an in-memory shared store shared by two Servers. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class ServerLoadDirectorTests + { + private const string BalancingUrl = "opc.tcp://balance:4840"; + + [Test] + public async Task RedirectsToLessLoadedPeerOnBalancingUrlAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions { BalancingEndpointUrl = BalancingUrl }; + var time = new FakeTimeProvider(); + + await PublishPeerAsync(store, context, options, time, "urn:B", serviceLevel: 255, load: 0); + + ServerLoadDirector director = CreateDirector( + store, context, options, time, "urn:A", localServiceLevel: 255, localLoad: 200); + + (bool redirect, ArrayOf endpoints) = + await director.TryGetDirectedEndpointsAsync(BalancingUrl, LocalEndpoints("urn:A")); + + Assert.That(redirect, Is.True); + EndpointDescription[] result = endpoints.ToArray(); + Assert.That(result, Has.Length.EqualTo(1)); + Assert.That(result[0].Server.ApplicationUri, Is.EqualTo("urn:B")); + } + + [Test] + public async Task ServesLocalOnNormalUrlAndPublishesOwnEndpointsAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions { BalancingEndpointUrl = BalancingUrl }; + var time = new FakeTimeProvider(); + + ServerLoadDirector director = CreateDirector( + store, context, options, time, "urn:A", localServiceLevel: 255, localLoad: 0); + + (bool redirect, _) = await director.TryGetDirectedEndpointsAsync( + "opc.tcp://a:4840", LocalEndpoints("urn:A")); + + Assert.That(redirect, Is.False, "a normal discovery request is served locally"); + + var directory = new SharedPeerEndpointDirectory( + store, context, NullRecordProtector.Instance, options); + EndpointDescription[] published = (await directory.GetEndpointsAsync("urn:A")).ToArray(); + Assert.That(published, Has.Length.EqualTo(1), "the local endpoints are published for peers"); + Assert.That(published[0].Server.ApplicationUri, Is.EqualTo("urn:A")); + } + + [Test] + public async Task RedirectsToHealthierActivePeerRegardlessOfLoadAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions { BalancingEndpointUrl = BalancingUrl }; + var time = new FakeTimeProvider(); + + await PublishPeerAsync(store, context, options, time, "urn:B", serviceLevel: 255, load: 250); + + // Local Server is a cold standby (NoData). + ServerLoadDirector director = CreateDirector( + store, context, options, time, "urn:A", localServiceLevel: ServiceLevels.NoData, localLoad: 0); + + (bool redirect, ArrayOf endpoints) = + await director.TryGetDirectedEndpointsAsync(BalancingUrl, LocalEndpoints("urn:A")); + + Assert.That(redirect, Is.True); + Assert.That(endpoints.ToArray()[0].Server.ApplicationUri, Is.EqualTo("urn:B")); + } + + [Test] + public async Task ServesLocalWhenNotConfiguredAsync() + { + var options = new LoadDirectionOptions { BalancingEndpointUrl = BalancingUrl }; + var director = new ServerLoadDirector( + new ConstantServiceLevelProvider(255), new ConstantLoadWeightProvider(0), options); + + (bool redirect, _) = await director.TryGetDirectedEndpointsAsync(BalancingUrl, LocalEndpoints("urn:A")); + + Assert.That(redirect, Is.False, "an unconfigured director never redirects"); + } + + [Test] + public async Task ServesLocalWhenTargetEndpointsMissingAsync() + { + using var store = new InMemorySharedKeyValueStore(); + IServiceMessageContext context = CreateContext(); + var options = new LoadDirectionOptions { BalancingEndpointUrl = BalancingUrl }; + var time = new FakeTimeProvider(); + + // B is the best target by health/load but never published its endpoints. + var directionPublisher = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, "urn:B"); + await directionPublisher.PublishServiceLevelAsync(255); + await directionPublisher.PublishLoadWeightAsync(0); + + ServerLoadDirector director = CreateDirector( + store, context, options, time, "urn:A", localServiceLevel: 255, localLoad: 200); + + (bool redirect, _) = await director.TryGetDirectedEndpointsAsync(BalancingUrl, LocalEndpoints("urn:A")); + + Assert.That(redirect, Is.False, "without the target's endpoints the request fails safe to the local Server"); + } + + private static ServerLoadDirector CreateDirector( + InMemorySharedKeyValueStore store, + IServiceMessageContext context, + LoadDirectionOptions options, + FakeTimeProvider time, + string localServerUri, + byte localServiceLevel, + byte localLoad) + { + var view = new SharedPeerDirectionView(store, context, NullRecordProtector.Instance, options, time); + var policy = new BandedServerDirectionPolicy(view, options, _ => 0); + var directory = new SharedPeerEndpointDirectory(store, context, NullRecordProtector.Instance, options); + var endpointPublisher = new SharedPeerEndpointPublisher( + store, context, NullRecordProtector.Instance, options, localServerUri); + + var director = new ServerLoadDirector( + new ConstantServiceLevelProvider(localServiceLevel), + new ConstantLoadWeightProvider(localLoad), + options); + director.Configure(policy, directory, endpointPublisher, localServerUri); + return director; + } + + private static async Task PublishPeerAsync( + InMemorySharedKeyValueStore store, + IServiceMessageContext context, + LoadDirectionOptions options, + FakeTimeProvider time, + string serverUri, + byte serviceLevel, + byte load) + { + var directionPublisher = new SharedPeerDirectionPublisher( + store, context, NullRecordProtector.Instance, options, time, serverUri); + await directionPublisher.PublishServiceLevelAsync(serviceLevel); + await directionPublisher.PublishLoadWeightAsync(load); + + var endpointPublisher = new SharedPeerEndpointPublisher( + store, context, NullRecordProtector.Instance, options, serverUri); + await endpointPublisher.PublishAsync(LocalEndpoints(serverUri)); + } + + private static ArrayOf LocalEndpoints(string serverUri) + { + return + [ + new EndpointDescription + { + EndpointUrl = "opc.tcp://" + serverUri + ":4840", + SecurityMode = MessageSecurityMode.None, + SecurityPolicyUri = SecurityPolicies.None, + Server = new ApplicationDescription + { + ApplicationUri = serverUri, + ApplicationType = ApplicationType.Server + } + } + ]; + } + + private static ServiceMessageContext CreateContext() + { + return ServiceMessageContext.CreateEmpty(NUnitTelemetryContext.Create()); + } + } +} diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServerRedundancyStartupTaskTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServerRedundancyStartupTaskTests.cs new file mode 100644 index 0000000000..9315460327 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServerRedundancyStartupTaskTests.cs @@ -0,0 +1,249 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for guard paths. + /// The live node population path is exercised by the hosted-server + /// end-to-end test. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class ServerRedundancyStartupTaskTests + { + [Test] + public void ConstructorThrowsOnNullOptions() + { + Assert.That(() => new ServerRedundancyStartupTask(null!), Throws.ArgumentNullException); + } + + [Test] + public void OptionsUseSingleServerDefaults() + { + var options = new ServerRedundancyOptions(); + + Assert.That(options.Mode, Is.EqualTo(RedundancySupport.None)); + Assert.That(options.PeerServerUris, Is.Empty); + Assert.That(options.CurrentServerId, Is.Not.Empty); + Assert.That(options.PeerServiceLevel, Is.EqualTo(ServiceLevels.Maximum)); + } + + [Test] + public void OnServerStartedThrowsOnNullServer() + { + var task = new ServerRedundancyStartupTask(new ServerRedundancyOptions()); + + Assert.That(async () => await task.OnServerStartedAsync(null!), Throws.ArgumentNullException); + } + + [Test] + public async Task OnServerStartedNoOpsWhenServerObjectMissingAsync() + { + var task = new ServerRedundancyStartupTask(new ServerRedundancyOptions()); + var server = new Mock(); + server.Setup(s => s.ServerObject).Returns((ServerObjectState)null!); + + await task.OnServerStartedAsync(server.Object); + + server.VerifyGet(s => s.ServerObject, Times.Once); + } + + [Test] + public async Task OnServerStartedLeavesSubtypeMembersAbsentForNoneAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + var task = new ServerRedundancyStartupTask(new ServerRedundancyOptions()); + + await task.OnServerStartedAsync(loaded.Server.Object); + + PropertyState currentServerId = + loaded.Manager.FindPredefinedNode>( + VariableIds.Server_ServerRedundancy_CurrentServerId); + PropertyState> serverUriArray = + loaded.Manager.FindPredefinedNode>>( + VariableIds.Server_ServerRedundancy_ServerUriArray); + Assert.That(currentServerId, Is.Not.Null); + Assert.That(loaded.Server.Object.ServerObject.ServerRedundancy!.TypeDefinitionId, + Is.EqualTo(ServerRedundancyTypeId)); + Assert.That(currentServerId!.NodeId, Is.EqualTo(VariableIds.Server_ServerRedundancy_CurrentServerId)); + Assert.That(currentServerId.Value, Is.Null); + Assert.That(serverUriArray, Is.Not.Null); + Assert.That(serverUriArray!.NodeId, Is.EqualTo(VariableIds.Server_ServerRedundancy_ServerUriArray)); + Assert.That(serverUriArray.Value, Is.Empty); + } + + [Test] + public async Task OnServerStartedAddsCurrentServerIdForTransparentModeAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + var options = new ServerRedundancyOptions + { + Mode = RedundancySupport.Transparent, + CurrentServerId = "replica-a" + }; + options.PeerServerUris.Add("urn:peer-a"); + var task = new ServerRedundancyStartupTask(options); + + await task.OnServerStartedAsync(loaded.Server.Object); + + PropertyState currentServerId = + loaded.Manager.FindPredefinedNode>( + VariableIds.Server_ServerRedundancy_CurrentServerId); + PropertyState> redundantServerArray = + loaded.Manager.FindPredefinedNode>>( + VariableIds.Server_ServerRedundancy_RedundantServerArray); + Assert.That(currentServerId!.Value, Is.EqualTo("replica-a")); + Assert.That(loaded.Server.Object.ServerObject.ServerRedundancy!.TypeDefinitionId, + Is.EqualTo(TransparentRedundancyTypeId)); + Assert.That(currentServerId.NodeId, Is.EqualTo(VariableIds.Server_ServerRedundancy_CurrentServerId)); + Assert.That(redundantServerArray!.Value[0].ServerId, Is.EqualTo("urn:peer-a")); + } + + [Test] + public async Task OnServerStartedAddsServerUriArrayForNonTransparentModeAsync() + { + using LoadedDiagnosticsServer loaded = await CreateLoadedServerAsync(); + var options = new ServerRedundancyOptions + { + Mode = RedundancySupport.Hot + }; + options.PeerServerUris.Add("urn:peer-a"); + options.PeerServerUris.Add("urn:peer-b"); + var task = new ServerRedundancyStartupTask(options); + + await task.OnServerStartedAsync(loaded.Server.Object); + + PropertyState> serverUriArray = + loaded.Manager.FindPredefinedNode>>( + VariableIds.Server_ServerRedundancy_ServerUriArray); + PropertyState> redundantServerArray = + loaded.Manager.FindPredefinedNode>>( + VariableIds.Server_ServerRedundancy_RedundantServerArray); + Assert.That(serverUriArray!.Value, Is.EqualTo(new[] { "urn:peer-a", "urn:peer-b" })); + Assert.That(loaded.Server.Object.ServerObject.ServerRedundancy!.TypeDefinitionId, + Is.EqualTo(NonTransparentRedundancyTypeId)); + Assert.That(serverUriArray.NodeId, Is.EqualTo(VariableIds.Server_ServerRedundancy_ServerUriArray)); + Assert.That(redundantServerArray!.Value[0].ServerId, Is.EqualTo("urn:peer-a")); + Assert.That(redundantServerArray.Value[1].ServerId, Is.EqualTo("urn:peer-b")); + } + + private static async Task CreateLoadedServerAsync() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + var namespaceUris = new NamespaceTable(); + namespaceUris.Append("http://opcfoundation.org/UA/"); + var serverUris = new StringTable(); + var typeTree = new TypeTable(namespaceUris); + var messageContext = new ServiceMessageContext(telemetry, EncodeableFactory.Create()) + { + NamespaceUris = namespaceUris, + ServerUris = serverUris + }; + + var server = new Mock(); + var coreNodeManager = new Mock(); + coreNodeManager.Setup(m => m.ImportNodesAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(new ValueTask()); + var masterNodeManager = new Mock(); + masterNodeManager.Setup(m => m.RemoveReferencesAsync( + It.IsAny>(), + It.IsAny())) + .Returns(new ValueTask()); + + server.Setup(s => s.Telemetry).Returns(telemetry); + server.Setup(s => s.NamespaceUris).Returns(namespaceUris); + server.Setup(s => s.ServerUris).Returns(serverUris); + server.Setup(s => s.TypeTree).Returns(typeTree); + server.Setup(s => s.MessageContext).Returns(messageContext); + server.Setup(s => s.CoreNodeManager).Returns(coreNodeManager.Object); + server.Setup(s => s.NodeManager).Returns(masterNodeManager.Object); + server.Setup(s => s.Factory).Returns(EncodeableFactory.Create()); + + var context = new ServerSystemContext(server.Object); + server.Setup(s => s.DefaultSystemContext).Returns(context); + + var configuration = new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration() + }; + var manager = new DiagnosticsNodeManager(server.Object, configuration, NullLogger.Instance); + await manager.CreateAddressSpaceAsync(new Dictionary>()); + ServerObjectState serverObject = manager.FindPredefinedNode(ObjectIds.Server); + server.Setup(s => s.ServerObject).Returns(serverObject); + server.Setup(s => s.DiagnosticsNodeManager).Returns(manager); + return new LoadedDiagnosticsServer(server, manager); + } + + private sealed class LoadedDiagnosticsServer : IDisposable + { + public LoadedDiagnosticsServer( + Mock server, + DiagnosticsNodeManager manager) + { + Server = server; + Manager = manager; + } + + public Mock Server { get; } + + public DiagnosticsNodeManager Manager { get; } + + public void Dispose() + { + Manager.Dispose(); + } + } + + private static readonly NodeId ServerRedundancyTypeId = new(2034); + private static readonly NodeId TransparentRedundancyTypeId = new(2036); + private static readonly NodeId NonTransparentRedundancyTypeId = new(2039); + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServiceLevelProviderTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServiceLevelProviderTests.cs new file mode 100644 index 0000000000..ff86074bee --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServiceLevelProviderTests.cs @@ -0,0 +1,293 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the service-level providers. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class ServiceLevelProviderTests + { + [Test] + public void ConstantProviderReturnsConfiguredLevel() + { + Assert.That(new ConstantServiceLevelProvider().GetServiceLevel(), Is.EqualTo(ServiceLevels.Maximum)); + Assert.That(new ConstantServiceLevelProvider(ServiceLevels.HealthyMinimum).GetServiceLevel(), + Is.EqualTo(ServiceLevels.HealthyMinimum)); + } + + [Test] + public void ConstantProviderAcceptsEventSubscriptions() + { + var provider = new ConstantServiceLevelProvider(250); + void Handler(byte _) + { + } + + provider.ServiceLevelChanged += Handler; + provider.ServiceLevelChanged -= Handler; + + Assert.That(provider.GetServiceLevel(), Is.EqualTo(250)); + } + + [Test] + public async Task LeaderProviderReportsLeaderLevelWhenLeaderAsync() + { + await using var leaderElection = new StaticLeaderElection(true); + await using var standbyElection = new StaticLeaderElection(false); + using var leader = new LeaderServiceLevelProvider(leaderElection); + using var standby = new LeaderServiceLevelProvider(standbyElection); + + Assert.That(leader.GetServiceLevel(), Is.EqualTo(ServiceLevels.Maximum)); + Assert.That(standby.GetServiceLevel(), Is.EqualTo(ServiceLevels.DegradedMaximum)); + } + + [Test] + public async Task LeaderProviderMapsStandbySubrangeFromFailoverModeAsync() + { + await using var coldElection = new StaticLeaderElection(false); + await using var warmElection = new StaticLeaderElection(false); + await using var hotElection = new StaticLeaderElection(false); + using var cold = new LeaderServiceLevelProvider(coldElection, RedundancySupport.Cold); + using var warm = new LeaderServiceLevelProvider(warmElection, RedundancySupport.Warm); + using var hot = new LeaderServiceLevelProvider(hotElection, RedundancySupport.Hot); + + Assert.That(cold.GetServiceLevel(), Is.EqualTo(ServiceLevels.NoData)); + Assert.That(warm.GetServiceLevel(), Is.EqualTo(ServiceLevels.DegradedMaximum)); + Assert.That(hot.GetServiceLevel(), Is.EqualTo(ServiceLevels.Maximum)); + } + + [Test] + public async Task LeaderProviderDecrementsHealthyLevelForLoadBalancingAsync() + { + await using var election = new StaticLeaderElection(true); + using var provider = new LeaderServiceLevelProvider( + election, + RedundancySupport.Hot, + getConnectedClientCount: () => 3); + + Assert.That(provider.GetServiceLevel(), Is.EqualTo((byte)(ServiceLevels.Maximum - 3))); + } + + [Test] + public async Task LeaderProviderAppliesHealthInputAsMaximumLevelAsync() + { + await using var election = new StaticLeaderElection(true); + using var provider = new LeaderServiceLevelProvider( + election, + RedundancySupport.Hot, + getHealthServiceLevel: () => ServiceLevels.DegradedMaximum); + + Assert.That(provider.GetServiceLevel(), Is.EqualTo(ServiceLevels.DegradedMaximum)); + } + + [Test] + public async Task LeaderProviderRaisesServiceLevelChangedOnTransitionAsync() + { + await using var election = new MutableLeaderElection(); + using var provider = new LeaderServiceLevelProvider(election, leaderLevel: 255, standbyLevel: 10); + byte? observed = null; + provider.ServiceLevelChanged += level => observed = level; + + election.Set(true); + Assert.That(observed, Is.EqualTo(ServiceLevels.Maximum)); + Assert.That(provider.GetServiceLevel(), Is.EqualTo(ServiceLevels.Maximum)); + + election.Set(false); + Assert.That(observed, Is.EqualTo((byte)10)); + Assert.That(provider.GetServiceLevel(), Is.EqualTo((byte)10)); + } + + [Test] + public async Task LeaderProviderManualServiceLevelOverridesCurrentRoleAsync() + { + await using var election = new MutableLeaderElection(); + using var provider = new LeaderServiceLevelProvider(election); + byte? observed = null; + provider.ServiceLevelChanged += level => observed = level; + + provider.SetServiceLevel(ServiceLevels.Maintenance); + + Assert.That(observed, Is.EqualTo(ServiceLevels.Maintenance)); + Assert.That(provider.GetServiceLevel(), Is.EqualTo(ServiceLevels.Maintenance)); + } + + [Test] + public async Task LeaderProviderHotAndMirroredStandbyReportsMaximumAsync() + { + await using var election = new StaticLeaderElection(false); + using var provider = new LeaderServiceLevelProvider( + election, + RedundancySupport.HotAndMirrored); + + Assert.That(provider.GetServiceLevel(), Is.EqualTo(ServiceLevels.Maximum)); + } + + [Test] + public async Task LeaderProviderHealthCapsLeaderLevelAsync() + { + await using var election = new StaticLeaderElection(true); + using var provider = new LeaderServiceLevelProvider( + election, + RedundancySupport.Hot, + getHealthServiceLevel: () => 200); + + Assert.That(provider.GetServiceLevel(), Is.EqualTo(200)); + } + + [Test] + public async Task LeaderProviderLoadDecrementClampsToHealthyMinimumAsync() + { + await using var election = new StaticLeaderElection(true); + using var provider = new LeaderServiceLevelProvider( + election, + RedundancySupport.Hot, + getConnectedClientCount: () => 300); + + byte result = provider.GetServiceLevel(); + Assert.That(result, Is.EqualTo(ServiceLevels.HealthyMinimum)); + } + + [Test] + public async Task LeaderProviderDoesNotApplyLoadBalancingToDegradedLevelAsync() + { + await using var election = new StaticLeaderElection(true); + using var provider = new LeaderServiceLevelProvider( + election, + RedundancySupport.Hot, + getConnectedClientCount: () => 10, + getHealthServiceLevel: () => ServiceLevels.DegradedMaximum); + + Assert.That(provider.GetServiceLevel(), Is.EqualTo(ServiceLevels.DegradedMaximum)); + } + + [Test] + public async Task LeaderProviderManualOverrideClearsOnTransitionAsync() + { + await using var election = new MutableLeaderElection(); + using var provider = new LeaderServiceLevelProvider(election, leaderLevel: 255, standbyLevel: 200); + provider.SetServiceLevel(ServiceLevels.DegradedMaximum); + Assert.That(provider.GetServiceLevel(), Is.EqualTo(ServiceLevels.DegradedMaximum)); + + election.Set(true); + + Assert.That(provider.GetServiceLevel(), Is.EqualTo(255)); + } + + [Test] + public async Task LeaderProviderConcurrentManualOverrideAndClearDoesNotProduceTornZero() + { + await using var election = new MutableLeaderElection(); + election.Set(true); + using var provider = new LeaderServiceLevelProvider(election, leaderLevel: 255, standbyLevel: 200); + using var cts = new CancellationTokenSource(); + int invalidLevel = -1; + + Task writer = Task.Run(() => + { + while (!cts.IsCancellationRequested) + { + provider.SetServiceLevel(ServiceLevels.DegradedMaximum); + election.Set(true); + } + }); + Task reader = Task.Run(() => + { + for (int ii = 0; ii < 100_000; ii++) + { + byte level = provider.GetServiceLevel(); + if (level != ServiceLevels.DegradedMaximum && level != ServiceLevels.Maximum) + { + Volatile.Write(ref invalidLevel, level); + cts.Cancel(); + break; + } + } + + cts.Cancel(); + }); + + await Task.WhenAll(writer, reader).ConfigureAwait(false); + + Assert.That( + Volatile.Read(ref invalidLevel), + Is.EqualTo(-1), + "Concurrent SetServiceLevel/leadership clear must not expose torn ServiceLevel values."); + } + + [Test] + public void ConstantProviderConstructorThrowsOnNullElection() + { + Assert.That( + () => new LeaderServiceLevelProvider(null!), + Throws.ArgumentNullException); + } + + private sealed class MutableLeaderElection : ILeaderElection + { + public bool IsLeader => Volatile.Read(ref m_isLeader) != 0; + + public event Action? LeadershipChanged; + + public void Set(bool isLeader) + { + Volatile.Write(ref m_isLeader, isLeader ? 1 : 0); + LeadershipChanged?.Invoke(isLeader); + } + + public ValueTask TryAcquireOrRenewAsync(CancellationToken ct = default) + { + return new ValueTask(IsLeader); + } + + public void Start() + { + } + + public ValueTask DisposeAsync() + { + return default; + } + + private int m_isLeader; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServiceLevelStartupTaskTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServiceLevelStartupTaskTests.cs new file mode 100644 index 0000000000..dec0a55bee --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Redundancy/ServiceLevelStartupTaskTests.cs @@ -0,0 +1,79 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for guard paths. The + /// happy-path update of the live Server.ServiceLevel node is + /// exercised by the hosted-server end-to-end test. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class ServiceLevelStartupTaskTests + { + [Test] + public void ConstructorThrowsOnNullProvider() + { + Assert.That(() => new ServiceLevelStartupTask(null!), Throws.ArgumentNullException); + } + + [Test] + public void OnServerStartedThrowsOnNullServer() + { + var task = new ServiceLevelStartupTask(new ConstantServiceLevelProvider()); + + Assert.That(async () => await task.OnServerStartedAsync(null!), Throws.ArgumentNullException); + } + + [Test] + public async Task OnServerStartedNoOpsWhenServerObjectMissingAsync() + { + var task = new ServiceLevelStartupTask(new ConstantServiceLevelProvider()); + var server = new Mock(); + server.Setup(s => s.ServerObject).Returns((ServerObjectState)null!); + + await task.OnServerStartedAsync(server.Object); + + server.VerifyGet(s => s.ServerObject, Times.Once); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/ReplicatedOptionsTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/ReplicatedOptionsTests.cs new file mode 100644 index 0000000000..0f142b4653 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/ReplicatedOptionsTests.cs @@ -0,0 +1,202 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Net; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using Moq; +using NUnit.Framework; + +namespace Opc.Ua.Redundancy.Server.Tests +{ + /// + /// Unit tests for the replicated gossip options (transport selection, + /// peers, and decoding limits). + /// + [TestFixture] + [Category("Distributed")] + public sealed class ReplicatedOptionsTests + { + [Test] + public void DefaultsAreSet() + { + var options = new ReplicatedAddressSpaceOptions(); + + Assert.That(options.ReplicaId, Is.Not.EqualTo(ReplicaId.Empty)); + Assert.That(options.TimeProvider, Is.SameAs(TimeProvider.System)); + Assert.That(options.TransportFactory, Is.Null); + Assert.That(options.MaxEntryCount, Is.GreaterThan(0)); + Assert.That(options.MaxPayloadBytes, Is.GreaterThan(0)); + } + + [Test] + public void SessionOptionsExposeFastReconnectDefault() + { + var options = new ReplicatedSessionOptions(); + + Assert.That(options.Session, Is.Not.Null); + Assert.That(options.Session.EnableFastReconnect, Is.False); + } + + [Test] + public void CreateReaderOptionsHonorsLimits() + { + var options = new ReplicatedAddressSpaceOptions + { + MaxEntryCount = 5, + MaxPayloadBytes = 99 + }; + + CrdtReaderOptions reader = options.CreateReaderOptions(); + + Assert.That(reader.MaxCollectionCount, Is.EqualTo(5)); + Assert.That(reader.MaxStringBytes, Is.EqualTo(99)); + } + + [Test] + public async Task UseTcpGossipConfiguresTransportFactoryAsync() + { + var options = new ReplicatedAddressSpaceOptions(); + options.AddPeer(new IPEndPoint(IPAddress.Loopback, 4999)); + options.AllowUnauthenticatedGossip = true; + options.UseTcpGossip(IPAddress.Loopback, 0, TimeSpan.FromMilliseconds(50)); + + Assert.That(options.TransportFactory, Is.Not.Null); + ITransport transport = options.CreateTransport(EmptyServices(), out InMemoryNetwork? network); + try + { + Assert.That(transport, Is.InstanceOf()); + Assert.That(network, Is.Null); + } + finally + { + await transport.DisposeAsync(); + if (network != null) + { + await network.DisposeAsync(); + } + } + } + + [Test] + public async Task UseUdpGossipConfiguresTransportFactoryAsync() + { + var options = new ReplicatedSessionOptions(); + options.AddPeer(new IPEndPoint(IPAddress.Loopback, 4998)); + options.AllowUnauthenticatedGossip = true; + options.UseUdpGossip(IPAddress.Loopback, 0); + + Assert.That(options.TransportFactory, Is.Not.Null); + ITransport transport = options.CreateTransport(EmptyServices(), out InMemoryNetwork? network); + try + { + Assert.That(transport, Is.InstanceOf()); + Assert.That(network, Is.Null); + } + finally + { + await transport.DisposeAsync(); + if (network != null) + { + await network.DisposeAsync(); + } + } + } + + [Test] + public async Task CreateTransportDefaultsToInProcessNetworkAsync() + { + var options = new ReplicatedAddressSpaceOptions(); + + ITransport transport = options.CreateTransport(EmptyServices(), out InMemoryNetwork? network); + try + { + Assert.That(transport, Is.InstanceOf()); + Assert.That(network, Is.Not.Null); + } + finally + { + await transport.DisposeAsync(); + if (network != null) + { + await network.DisposeAsync(); + } + } + } + + [Test] + public void UnauthenticatedTcpGossipFailsClosedByDefault() + { + var options = new ReplicatedAddressSpaceOptions(); + options.UseTcpGossip(IPAddress.Loopback, 0); + + Assert.That( + () => options.CreateTransport(EmptyServices(), out _), + Throws.InvalidOperationException + .With.Message.Contains(nameof(ReplicatedGossipOptions.AllowUnauthenticatedGossip))); + } + + [Test] + public void UnauthenticatedUdpGossipFailsClosedByDefault() + { + var options = new ReplicatedAddressSpaceOptions(); + options.UseUdpGossip(IPAddress.Loopback, 0); + + Assert.That( + () => options.CreateTransport(EmptyServices(), out _), + Throws.InvalidOperationException + .With.Message.Contains(nameof(ReplicatedGossipOptions.AllowUnauthenticatedGossip))); + } + + [Test] + public void AddPeerRejectsNull() + { + var options = new ReplicatedAddressSpaceOptions(); + Assert.That(() => options.AddPeer(null!), Throws.ArgumentNullException); + } + + [Test] + public void UseTcpGossipRejectsNullAddress() + { + var options = new ReplicatedAddressSpaceOptions(); + Assert.That(() => options.UseTcpGossip(null!, 0), Throws.ArgumentNullException); + } + + private static IServiceProvider EmptyServices() + { + return Mock.Of(); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Security/RecordProtectorTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Security/RecordProtectorTests.cs new file mode 100644 index 0000000000..c234b5c7dc --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Security/RecordProtectorTests.cs @@ -0,0 +1,286 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for and + /// : authenticated-encryption round-trip + /// and fail-closed rejection of tampered, wrong-key, and malformed records. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class RecordProtectorTests + { + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(seed + i); + } + return key; + } + + [Test] + public void ProtectUnprotectRoundTrips() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(1)); + ByteString plaintext = ByteString.From(new byte[] { 10, 20, 30, 40, 50 }); + + ByteString sealed1 = protector.Protect(plaintext); + bool ok = protector.TryUnprotect(sealed1, out ByteString recovered); + + Assert.That(ok, Is.True); + Assert.That(recovered.ToArray(), Is.EqualTo(plaintext.ToArray())); + Assert.That(sealed1.ToArray(), Is.Not.EqualTo(plaintext.ToArray())); + } + + [Test] + public void EmptyPayloadRoundTrips() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(2)); + ByteString plaintext = ByteString.From(Array.Empty()); + + ByteString sealed1 = protector.Protect(plaintext); + bool ok = protector.TryUnprotect(sealed1, out ByteString recovered); + + Assert.That(ok, Is.True); + Assert.That(recovered.ToArray(), Is.Empty); + } + + [Test] + public void ProtectProducesDistinctCiphertextPerCall() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(3)); + ByteString plaintext = ByteString.From(new byte[] { 1, 2, 3 }); + + ByteString a = protector.Protect(plaintext); + ByteString b = protector.Protect(plaintext); + + // Random IV per call => different envelopes for identical plaintext. + Assert.That(a.ToArray(), Is.Not.EqualTo(b.ToArray())); + } + + [Test] + public void TamperedCiphertextIsRejected() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(4)); + ByteString sealed1 = protector.Protect(ByteString.From(new byte[] { 7, 7, 7, 7 })); + + byte[] tampered = sealed1.ToArray(); + // Flip a byte inside the ciphertext region (after the 21-byte header). + tampered[25] ^= 0xFF; + + bool ok = protector.TryUnprotect(ByteString.From(tampered), out ByteString recovered); + + Assert.That(ok, Is.False); + Assert.That(recovered.IsNull, Is.True); + } + + [Test] + public void TamperedTagIsRejected() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(5)); + ByteString sealed1 = protector.Protect(ByteString.From(new byte[] { 9, 9 })); + + byte[] tampered = sealed1.ToArray(); + tampered[tampered.Length - 1] ^= 0x01; + + bool ok = protector.TryUnprotect(ByteString.From(tampered), out _); + + Assert.That(ok, Is.False); + } + + [Test] + public void TamperedIvIsRejected() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(6)); + ByteString sealed1 = protector.Protect(ByteString.From(new byte[] { 4, 5, 6 })); + + byte[] tampered = sealed1.ToArray(); + // IV occupies bytes [5, 21); the MAC covers it, so a flip is caught. + tampered[6] ^= 0x80; + + bool ok = protector.TryUnprotect(ByteString.From(tampered), out _); + + Assert.That(ok, Is.False); + } + + [Test] + public void WrongMasterKeyIsRejected() + { + using var writer = new AesCbcHmacRecordProtector(MakeKey(7)); + using var reader = new AesCbcHmacRecordProtector(MakeKey(8)); + ByteString sealed1 = writer.Protect(ByteString.From(new byte[] { 1, 1, 1, 1 })); + + bool ok = reader.TryUnprotect(sealed1, out _); + + Assert.That(ok, Is.False); + } + + [Test] + public void DifferentKeyIdIsRejected() + { + byte[] key = MakeKey(9); + using var writer = new AesCbcHmacRecordProtector(key, keyId: 1); + using var reader = new AesCbcHmacRecordProtector(key, keyId: 2); + ByteString sealed1 = writer.Protect(ByteString.From(new byte[] { 2, 2 })); + + bool ok = reader.TryUnprotect(sealed1, out _); + + Assert.That(ok, Is.False); + } + + [Test] + public void MalformedOrNullEnvelopeIsRejected() + { + using var protector = new AesCbcHmacRecordProtector(MakeKey(10)); + + Assert.That(protector.TryUnprotect(default, out _), Is.False); + Assert.That(protector.TryUnprotect(ByteString.From(new byte[] { 1, 2, 3 }), out _), Is.False); + } + + [Test] + public void ConstructorRejectsShortMasterKey() + { + Assert.That( + () => new AesCbcHmacRecordProtector(new byte[16]), + Throws.ArgumentException); + } + + [Test] + public void NullProtectorPassesThrough() + { + NullRecordProtector protector = NullRecordProtector.Instance; + ByteString plaintext = ByteString.From(new byte[] { 3, 1, 4, 1, 5 }); + + ByteString sealed1 = protector.Protect(plaintext); + bool ok = protector.TryUnprotect(sealed1, out ByteString recovered); + + Assert.That(ok, Is.True); + Assert.That(sealed1.ToArray(), Is.EqualTo(plaintext.ToArray())); + Assert.That(recovered.ToArray(), Is.EqualTo(plaintext.ToArray())); + } + } + + /// + /// Unit tests for : staged key rotation + /// where new writes use the active key while reads still verify against + /// retired keys. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class KeyRingRecordProtectorTests + { + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(seed + i); + } + return key; + } + + [Test] + public void ActiveKeyRoundTrips() + { + using var active = new AesCbcHmacRecordProtector(MakeKey(30), keyId: 2); + using var ring = new KeyRingRecordProtector(active); + ByteString plaintext = ByteString.From(new byte[] { 1, 2, 3 }); + + ByteString sealed1 = ring.Protect(plaintext); + bool ok = ring.TryUnprotect(sealed1, out ByteString recovered); + + Assert.That(ok, Is.True); + Assert.That(recovered.ToArray(), Is.EqualTo(plaintext.ToArray())); + } + + [Test] + public void RetiredKeyStillReadsAfterRotation() + { + using var oldKey = new AesCbcHmacRecordProtector(MakeKey(31), keyId: 1); + using var newKey = new AesCbcHmacRecordProtector(MakeKey(32), keyId: 2); + ByteString plaintext = ByteString.From(new byte[] { 9, 9, 9 }); + + // A record written before rotation, under the old key. + ByteString legacyRecord = oldKey.Protect(plaintext); + + // After rotation the ring writes under the new key but still reads + // records produced under the retired key. + using var ring = new KeyRingRecordProtector(newKey, oldKey); + bool ok = ring.TryUnprotect(legacyRecord, out ByteString recovered); + + Assert.That(ok, Is.True); + Assert.That(recovered.ToArray(), Is.EqualTo(plaintext.ToArray())); + } + + [Test] + public void NewWritesUseActiveKeyOnly() + { + using var oldKey = new AesCbcHmacRecordProtector(MakeKey(33), keyId: 1); + using var newKey = new AesCbcHmacRecordProtector(MakeKey(34), keyId: 2); + using var ring = new KeyRingRecordProtector(newKey, oldKey); + + ByteString sealed1 = ring.Protect(ByteString.From(new byte[] { 5 })); + + // The retired key alone must not be able to read a post-rotation record. + Assert.That(oldKey.TryUnprotect(sealed1, out _), Is.False); + Assert.That(newKey.TryUnprotect(sealed1, out _), Is.True); + } + + [Test] + public void RecordOutsideRingIsRejected() + { + using var member = new AesCbcHmacRecordProtector(MakeKey(35), keyId: 1); + using var stranger = new AesCbcHmacRecordProtector(MakeKey(36), keyId: 9); + using var ring = new KeyRingRecordProtector(member); + + ByteString foreignRecord = stranger.Protect(ByteString.From(new byte[] { 7, 7 })); + + Assert.That(ring.TryUnprotect(foreignRecord, out _), Is.False); + } + + [Test] + public void ConstructorRejectsNullActive() + { + Assert.That(() => new KeyRingRecordProtector(null!), Throws.ArgumentNullException); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/CrdtSessionManagerFactoryTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/CrdtSessionManagerFactoryTests.cs new file mode 100644 index 0000000000..a9357b36e7 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/CrdtSessionManagerFactoryTests.cs @@ -0,0 +1,205 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using NUnit.Framework; +using Opc.Ua.Security.Certificates; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Hosting; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server.Tests +{ + /// + /// Tests for the CRDT session DI seam: + /// and the UseReplicatedSessions / UseReplicatedAddressSpace + /// fluent registrations. + /// + [TestFixture] + [Category("Distributed")] + public sealed class CrdtSessionManagerFactoryTests + { + [Test] + public async Task FactoryCreatesDistributedSessionManagerAsync() + { + // In-process gossip unit tests knowingly opt out of record protection. + await using ServiceProvider services = ServicesWithNullProtector(); + + await using var factory = new CrdtSessionManagerFactory( + services, new ReplicatedSessionOptions()); + + using var manager = factory.Create( + NewServer().Object, + NewConfiguration(), + TimeProvider.System, + _ => (Certificate?)null) as DistributedSessionManager; + + Assert.That(manager, Is.Not.Null); + } + + [Test] + public void ConstructorRejectsNullArguments() + { + Assert.That( + () => new CrdtSessionManagerFactory(null!, new ReplicatedSessionOptions()), + Throws.ArgumentNullException); + Assert.That( + () => new CrdtSessionManagerFactory(Mock.Of(), null!), + Throws.ArgumentNullException); + } + + [Test] + public async Task FactoryRejectsReplicatedSessionStoreWithoutProtectorAsync() + { + await using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + await using var factory = new CrdtSessionManagerFactory( + services, new ReplicatedSessionOptions()); + + Assert.That( + () => factory.Create( + NewServer().Object, + NewConfiguration(), + TimeProvider.System, + _ => (Certificate?)null), + Throws.InvalidOperationException + .With.Message.Contains(nameof(IRecordProtector))); + } + + [Test] + public async Task FactoryRejectsFastReconnectWithoutStronglyConsistentNonceStoreAsync() + { + // Fast reconnect replays a session by AuthenticationToken; the + // single-use nonce must be strongly consistent across the replica + // set. A record protector alone (no shared nonce store) must fail + // closed rather than silently use a per-process registry. + await using ServiceProvider services = ServicesWithNullProtector(); + var options = new ReplicatedSessionOptions(); + options.Session.EnableFastReconnect = true; + await using var factory = new CrdtSessionManagerFactory(services, options); + + Assert.That( + () => factory.Create( + NewServer().Object, + NewConfiguration(), + TimeProvider.System, + _ => (Certificate?)null), + Throws.InvalidOperationException + .With.Message.Contains(nameof(ISharedKeyValueStore))); + } + + [Test] + public async Task FactoryCreatesWithFastReconnectAndRegisteredNonceStoreAsync() + { + await using ServiceProvider services = new ServiceCollection() + .AddSingleton(NullRecordProtector.Instance) + .AddSingleton(new InMemorySharedKeyValueStore()) + .BuildServiceProvider(); + var options = new ReplicatedSessionOptions(); + options.Session.EnableFastReconnect = true; + await using var factory = new CrdtSessionManagerFactory(services, options); + + using var manager = factory.Create( + NewServer().Object, + NewConfiguration(), + TimeProvider.System, + _ => (Certificate?)null) as DistributedSessionManager; + + Assert.That(manager, Is.Not.Null); + } + + [Test] + public async Task UseReplicatedSessionsRegistersFactoryAsync() + { + var services = new ServiceCollection(); + // In-process gossip unit tests knowingly opt out of record protection. + services.AddSingleton(NullRecordProtector.Instance); + services.AddOpcUa() + .AddServer(_ => { }) + .UseReplicatedSessions(o => o.Session.EnableFastReconnect = true); + await using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + } + + [Test] + public async Task UseReplicatedAddressSpaceRegistersStartupTaskAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseReplicatedAddressSpace(); + await using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That(provider.GetServices(), + Has.Some.InstanceOf()); + } + + private static ServiceProvider ServicesWithNullProtector() + { + return new ServiceCollection() + .AddSingleton(NullRecordProtector.Instance) + .BuildServiceProvider(); + } + + private static Mock NewServer() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + IServiceMessageContext context = ServiceMessageContext.CreateEmpty(telemetry); + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(telemetry); + server.Setup(s => s.MessageContext).Returns(context); + return server; + } + + private static ApplicationConfiguration NewConfiguration() + { + return new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration + { + MinSessionTimeout = 1000, + MaxSessionTimeout = 3_600_000, + MaxSessionCount = 100 + } + }; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/CrdtSharedKeyValueStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/CrdtSharedKeyValueStoreTests.cs new file mode 100644 index 0000000000..4a0de06d29 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/CrdtSharedKeyValueStoreTests.cs @@ -0,0 +1,153 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using NUnit.Framework; + +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server.Tests +{ + /// + /// Tests for the CRDT-backed used to + /// replicate mirrored session entries active/active. + /// + [TestFixture] + [Category("Distributed")] + public sealed class CrdtSharedKeyValueStoreTests + { + [Test] + public async Task ReplicatesSetAndDeleteAsync() + { + await using var network = new InMemoryNetwork(); + await using var storeA = CreateStore(network, 1); + await using var storeB = CreateStore(network, 2); + + // Warm up both transports so neither misses the other's broadcasts. + await storeA.TryGetAsync("warmup"); + await storeB.TryGetAsync("warmup"); + + var value = new ByteString(new byte[] { 1, 2, 3, 4 }); + await storeA.SetAsync("session/abc", value); + + await AssertEventuallyAsync( + async () => + { + (bool found, ByteString stored) = await storeB.TryGetAsync("session/abc"); + return found && stored.ToArray().AsSpan().SequenceEqual(value.ToArray()); + }, + "a value set on A should replicate to B"); + + await storeA.DeleteAsync("session/abc"); + + await AssertEventuallyAsync( + async () => + { + (bool found, _) = await storeB.TryGetAsync("session/abc"); + return !found; + }, + "a delete on A should replicate to B"); + } + + [Test] + public async Task CompareAndSwapThrowsNotSupportedAsync() + { + await using var network = new InMemoryNetwork(); + await using var store = CreateStore(network, 1); + + Assert.That( + async () => await store.CompareAndSwapAsync("k", default, new ByteString(new byte[] { 1 })), + Throws.TypeOf(), + "CRDT stores cannot provide a linearizable compare-and-swap"); + } + + [Test] + public async Task ScanReturnsOnlyEntriesWithMatchingPrefixAsync() + { + await using var network = new InMemoryNetwork(); + await using var store = CreateStore(network, 1); + + await store.SetAsync("session/a", new ByteString(new byte[] { 1 })); + await store.SetAsync("session/b", new ByteString(new byte[] { 2 })); + await store.SetAsync("other/c", new ByteString(new byte[] { 3 })); + + var found = new System.Collections.Generic.List(); + await foreach (System.Collections.Generic.KeyValuePair pair in + store.ScanAsync("session/")) + { + found.Add(pair.Key); + } + + Assert.That(found, Is.EquivalentTo(s_sessionKeys)); + } + + [Test] + public async Task WatchThrowsNotSupported() + { + await using var network = new InMemoryNetwork(); + await using var store = CreateStore(network, 1); + + Assert.That(() => store.WatchAsync("session/"), Throws.TypeOf()); + } + + private static CrdtSharedKeyValueStore CreateStore(InMemoryNetwork network, ulong replica) + { + return new CrdtSharedKeyValueStore( + ReplicaId.FromUInt64(replica), + network.CreateTransport(), + TimeProvider.System, + CrdtReaderOptions.Default); + } + + private static async Task AssertEventuallyAsync(Func> condition, string message) + { + // Generous deadline: CRDT convergence over the in-memory gossip network is normally + // sub-second, but background loops can be CPU-starved on a loaded CI runner. + DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline) + { + if (await condition()) + { + return; + } + await Task.Delay(25); + } + Assert.Fail(message); + } + + private static readonly string[] s_sessionKeys = ["session/a", "session/b"]; + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionFactoryTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionFactoryTests.cs new file mode 100644 index 0000000000..67687fd2a4 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionFactoryTests.cs @@ -0,0 +1,142 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using NUnit.Framework; +using Opc.Ua.Security.Certificates; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the distributed session DI seam: + /// and the + /// UseDistributedSessions fluent registration. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public sealed class DistributedSessionFactoryTests + { + [Test] + public void FactoryCreatesDistributedSessionManager() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + IServiceMessageContext context = ServiceMessageContext.CreateEmpty(telemetry); + var serverMock = new Mock(); + serverMock.Setup(s => s.Telemetry).Returns(telemetry); + serverMock.Setup(s => s.MessageContext).Returns(context); + + var configuration = new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration + { + MinSessionTimeout = 1000, + MaxSessionTimeout = 3_600_000, + MaxSessionCount = 100 + } + }; + + using var kv = new InMemorySharedKeyValueStore(); + var factory = new DistributedSessionManagerFactory(kv); + + using var manager = factory.Create( + serverMock.Object, + configuration, + TimeProvider.System, + _ => (Certificate?)null) as DistributedSessionManager; + + Assert.That(manager, Is.Not.Null); + } + + [Test] + public void FactoryConstructorRejectsNullStore() + { + Assert.That( + () => new DistributedSessionManagerFactory(null!), + Throws.ArgumentNullException); + } + + [Test] + public async Task UseDistributedSessionsRegistersFactoryAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseDistributedSessions(o => o.EnableFastReconnect = true); + await using ServiceProvider provider = services.BuildServiceProvider(); + + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + } + + [Test] + public async Task UseDistributedSessionsSharesStoreWithAddressSpaceAsync() + { + var services = new ServiceCollection(); + services.AddOpcUa() + .AddServer(_ => { }) + .UseDistributedAddressSpace() + .UseDistributedSessions(); + await using ServiceProvider provider = services.BuildServiceProvider(); + + ISharedKeyValueStore first = provider.GetRequiredService(); + ISharedKeyValueStore second = provider.GetRequiredService(); + + // Both features compose over a single shared backend. + Assert.That(ReferenceEquals(first, second), Is.True); + Assert.That( + provider.GetRequiredService(), + Is.InstanceOf()); + } + + [Test] + public void DefaultOptionsDisableFastReconnect() + { + var options = new DistributedSessionOptions(); + + Assert.That(options.EnableFastReconnect, Is.False); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionManagerTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionManagerTests.cs new file mode 100644 index 0000000000..68fdee5813 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionManagerTests.cs @@ -0,0 +1,208 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Security.Certificates; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for the security decision logic of + /// : the SecurityPolicy/Mode + /// check and the single-use server-nonce consumption (replay defence) that + /// gate a mirrored fast-reconnect. The full reconstruct + signature path is + /// exercised by the two-server end-to-end test. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class DistributedSessionManagerTests + { + private const string PolicyA = "http://opcfoundation.org/UA/SecurityPolicy#Basic256Sha256"; + private const string PolicyB = "http://opcfoundation.org/UA/SecurityPolicy#Aes256_Sha256_RsaPss"; + + private static readonly InMemorySharedKeyValueStore s_sessionKv = new(); + private static readonly SharedKeyValueSessionStore s_sessionStore = + new(s_sessionKv, ServiceMessageContext.CreateEmpty(NUnitTelemetryContext.Create())); + + private static DistributedSessionManager CreateManager(ISingleUseNonceRegistry nonceRegistry) + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + + var serverMock = new Mock(); + serverMock.Setup(s => s.Telemetry).Returns(telemetry); + + var configuration = new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration + { + MinSessionTimeout = 1000, + MaxSessionTimeout = 3_600_000, + MaxSessionCount = 100, + MaxRequestAge = 60_000, + MaxBrowseContinuationPoints = 10, + MaxHistoryContinuationPoints = 10 + } + }; + + return new DistributedSessionManager( + serverMock.Object, + configuration, + s_sessionStore, + nonceRegistry, + _ => (Certificate?)null, + new DistributedSessionOptions { EnableFastReconnect = true }); + } + + private static SharedSessionEntry EntryWithNonce(byte[] nonce) + { + return new SharedSessionEntry + { + SessionId = new NodeId(1, 1), + AuthenticationToken = new NodeId(2, 1), + SecurityPolicyUri = PolicyA, + SecurityMode = (int)MessageSecurityMode.SignAndEncrypt, + ServerNonce = ByteString.From(nonce) + }; + } + + [Test] + public async Task AuthorizeSucceedsForMatchingPolicyAndFreshNonceAsync() + { + using var registryKv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(registryKv); + using DistributedSessionManager manager = CreateManager(registry); + SharedSessionEntry entry = EntryWithNonce(new byte[] { 1, 2, 3, 4 }); + + DistributedSessionManager.RestoreDecision decision = await manager.AuthorizeAndConsumeAsync( + entry, PolicyA, MessageSecurityMode.SignAndEncrypt); + + Assert.That(decision, Is.EqualTo(DistributedSessionManager.RestoreDecision.Authorized)); + } + + [Test] + public async Task AuthorizeRejectsMismatchedPolicyAsync() + { + using var registryKv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(registryKv); + using DistributedSessionManager manager = CreateManager(registry); + SharedSessionEntry entry = EntryWithNonce(new byte[] { 1, 2, 3, 4 }); + + DistributedSessionManager.RestoreDecision decision = await manager.AuthorizeAndConsumeAsync( + entry, PolicyB, MessageSecurityMode.SignAndEncrypt); + + Assert.That(decision, Is.EqualTo(DistributedSessionManager.RestoreDecision.PolicyMismatch)); + } + + [Test] + public async Task AuthorizeRejectsMismatchedSecurityModeAsync() + { + using var registryKv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(registryKv); + using DistributedSessionManager manager = CreateManager(registry); + SharedSessionEntry entry = EntryWithNonce(new byte[] { 1, 2, 3, 4 }); + + DistributedSessionManager.RestoreDecision decision = await manager.AuthorizeAndConsumeAsync( + entry, PolicyA, MessageSecurityMode.Sign); + + Assert.That(decision, Is.EqualTo(DistributedSessionManager.RestoreDecision.PolicyMismatch)); + } + + [Test] + public async Task AuthorizeRejectsReplayedNonceAsync() + { + using var registryKv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(registryKv); + using DistributedSessionManager manager = CreateManager(registry); + SharedSessionEntry entry = EntryWithNonce(new byte[] { 9, 9, 9, 9 }); + + DistributedSessionManager.RestoreDecision first = await manager.AuthorizeAndConsumeAsync( + entry, PolicyA, MessageSecurityMode.SignAndEncrypt); + DistributedSessionManager.RestoreDecision second = await manager.AuthorizeAndConsumeAsync( + entry, PolicyA, MessageSecurityMode.SignAndEncrypt); + + Assert.That(first, Is.EqualTo(DistributedSessionManager.RestoreDecision.Authorized)); + Assert.That(second, Is.EqualTo(DistributedSessionManager.RestoreDecision.NonceReplayed), + "a captured activation cannot be replayed once the nonce is consumed"); + } + + [Test] + public async Task AuthorizeRejectsEmptyNonceAsync() + { + using var registryKv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(registryKv); + using DistributedSessionManager manager = CreateManager(registry); + SharedSessionEntry entry = EntryWithNonce(Array.Empty()); + + DistributedSessionManager.RestoreDecision decision = await manager.AuthorizeAndConsumeAsync( + entry, PolicyA, MessageSecurityMode.SignAndEncrypt); + + Assert.That(decision, Is.EqualTo(DistributedSessionManager.RestoreDecision.NonceReplayed)); + } + + [Test] + public void ConstructorValidatesArguments() + { + using var registryKv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(registryKv); + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + IServiceMessageContext context = ServiceMessageContext.CreateEmpty(telemetry); + var serverMock = new Mock(); + serverMock.Setup(s => s.Telemetry).Returns(telemetry); + var configuration = new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration + { + MinSessionTimeout = 1000, + MaxSessionTimeout = 3_600_000, + MaxSessionCount = 100 + } + }; + using var kv = new InMemorySharedKeyValueStore(); + var store = new SharedKeyValueSessionStore(kv, context); + + Assert.That( + () => new DistributedSessionManager( + serverMock.Object, configuration, store, registry, null!), + Throws.ArgumentNullException); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs new file mode 100644 index 0000000000..320f1cd457 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionMirrorIntegrationTests.cs @@ -0,0 +1,135 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.TestFramework; +using Quickstarts.ReferenceServer; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Integration test that drives a real, fully-started server whose + /// builds a + /// , and verifies that a session + /// created and activated through the real service handlers is mirrored + /// (encrypted) to the shared store and removed again on close. This closes + /// the runtime-wiring gap (factory -> StandardServer.CreateSessionManager + /// -> mirror on real CreateSession / ActivateSession / close) + /// that the manager unit tests cannot exercise. The secured token-reuse + /// restore path is covered by the manager's unit tests (policy match + + /// single-use nonce) and the base activation integration tests. + /// + [TestFixture] + [Category("Distributed")] + [Category("Session")] + [NonParallelizable] + public class DistributedSessionMirrorIntegrationTests + { + private InMemorySharedKeyValueStore m_kv = null!; + private AesCbcHmacRecordProtector m_protector = null!; + private ServerFixture m_fixture = null!; + private StandardServer m_server = null!; + + [OneTimeSetUp] + public async Task OneTimeSetUpAsync() + { + m_kv = new InMemorySharedKeyValueStore(); + m_protector = new AesCbcHmacRecordProtector(MakeKey(1)); + var factory = new DistributedSessionManagerFactory( + m_kv, m_protector, new DistributedSessionOptions { EnableFastReconnect = true }); + + m_fixture = new ServerFixture(t => + { + var server = new ReferenceServer(t); + server.SessionManagerFactory = factory; + return server; + }); + m_server = await m_fixture.StartAsync(); + } + + [OneTimeTearDown] + public async Task OneTimeTearDownAsync() + { + if (m_fixture != null) + { + await m_fixture.StopAsync(); + } + m_protector?.Dispose(); + m_kv?.Dispose(); + } + + [Test] + public async Task SessionIsMirroredEncryptedOnActivateAndRemovedOnCloseAsync() + { + const string sessionName = nameof(SessionIsMirroredEncryptedOnActivateAndRemovedOnCloseAsync); + + (RequestHeader header, SecureChannelContext context) = + await m_server.CreateAndActivateSessionAsync(sessionName); + + IServiceMessageContext messageContext = m_server.CurrentInstance.MessageContext; + var store = new SharedKeyValueSessionStore(m_kv, messageContext, m_protector); + + SharedSessionEntry? entry = await store.TryGetAsync(header.AuthenticationToken); + + Assert.That(entry, Is.Not.Null, "the session must be mirrored to the shared store on activate"); + Assert.That(entry!.AuthenticationToken, Is.EqualTo(header.AuthenticationToken)); + + // Encrypted at rest: a protector with a different key fails closed. + using var wrongKey = new AesCbcHmacRecordProtector(MakeKey(2)); + var wrongStore = new SharedKeyValueSessionStore(m_kv, messageContext, wrongKey); + SharedSessionEntry? wrong = await wrongStore.TryGetAsync(header.AuthenticationToken); + Assert.That(wrong, Is.Null, "the mirrored entry is encrypted; the wrong key cannot read it"); + + await m_server.CloseSessionAsync(context, header, true, RequestLifetime.None); + + SharedSessionEntry? afterClose = await store.TryGetAsync(header.AuthenticationToken); + Assert.That(afterClose, Is.Null, "the mirror must be removed when the session closes"); + } + + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(seed + i); + } + return key; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionTakeoverIntegrationTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionTakeoverIntegrationTests.cs new file mode 100644 index 0000000000..150d6e59e5 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/DistributedSessionTakeoverIntegrationTests.cs @@ -0,0 +1,213 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.TestFramework; +using Quickstarts.ReferenceServer; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Two-replica session takeover tests for distributed session mirroring. + /// + [TestFixture] + [Category("Distributed")] + [Category("Session")] + [NonParallelizable] + public class DistributedSessionTakeoverIntegrationTests + { + [Test] + public async Task SessionCreatedOnActiveCanActivateOnBackupAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var factory = new DistributedSessionManagerFactory( + kv, + options: new DistributedSessionOptions { EnableFastReconnect = true }); + var activeFixture = new ServerFixture(t => + { + var server = new ReferenceServer(t); + server.SessionManagerFactory = factory; + return server; + }) + { + SecurityNone = true + }; + var backupFixture = new ServerFixture(t => + { + var server = new ReferenceServer(t); + server.SessionManagerFactory = factory; + return server; + }) + { + SecurityNone = true + }; + + StandardServer? active = null; + StandardServer? backup = null; + try + { + active = await activeFixture.StartAsync(); + backup = await backupFixture.StartAsync(); + (RequestHeader activeHeader, _) = await active.CreateAndActivateSessionAsync( + nameof(SessionCreatedOnActiveCanActivateOnBackupAsync)); + + EndpointDescription backupEndpoint = backup.GetEndpoints() + .Find(e => e.SecurityMode == MessageSecurityMode.None)!; + var backupChannelContext = new SecureChannelContext( + "backup-channel", + backupEndpoint, + RequestEncoding.Binary, + null, + null, + null); + var takeoverHeader = new RequestHeader + { + AuthenticationToken = activeHeader.AuthenticationToken + }; + + ActivateSessionResponse response = await backup.ActivateSessionAsync( + backupChannelContext, + takeoverHeader, + null, + [], + [], + default, + null, + RequestLifetime.None); + + ServerFixtureUtils.ValidateResponse(response.ResponseHeader); + Assert.That(response.ServerNonce.IsNull, Is.False); + + await backup.CloseSessionAsync( + backupChannelContext, + takeoverHeader, + true, + RequestLifetime.None); + var mirror = new SharedKeyValueSessionStore(kv, backup.CurrentInstance.MessageContext); + SharedSessionEntry? afterClose = await mirror.TryGetAsync(activeHeader.AuthenticationToken); + Assert.That(afterClose, Is.Null, "backup ownership must close and remove the mirrored session"); + } + finally + { + if (backup != null) + { + await backupFixture.StopAsync(); + } + if (active != null) + { + await activeFixture.StopAsync(); + } + } + } + + [Test] + public async Task SessionTakeoverPreservesAuthenticationTokenAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var factory = new DistributedSessionManagerFactory(kv, + options: new DistributedSessionOptions { EnableFastReconnect = true }); + var activeFixture = new ServerFixture(t => + { + var server = new ReferenceServer(t); + server.SessionManagerFactory = factory; + return server; + }) + { + SecurityNone = true + }; + var backupFixture = new ServerFixture(t => + { + var server = new ReferenceServer(t); + server.SessionManagerFactory = factory; + return server; + }) + { + SecurityNone = true + }; + + StandardServer? active = null; + StandardServer? backup = null; + try + { + active = await activeFixture.StartAsync(); + backup = await backupFixture.StartAsync(); + (RequestHeader activeHeader, _) = await active.CreateAndActivateSessionAsync( + nameof(SessionTakeoverPreservesAuthenticationTokenAsync)); + + NodeId originalToken = activeHeader.AuthenticationToken; + EndpointDescription backupEndpoint = backup.GetEndpoints() + .Find(e => e.SecurityMode == MessageSecurityMode.None)!; + var backupChannelContext = new SecureChannelContext( + "backup-channel", + backupEndpoint, + RequestEncoding.Binary, + null, + null, + null); + var takeoverHeader = new RequestHeader + { + AuthenticationToken = originalToken + }; + + ActivateSessionResponse response = await backup.ActivateSessionAsync( + backupChannelContext, + takeoverHeader, + null, + [], + [], + default, + null, + RequestLifetime.None); + + ServerFixtureUtils.ValidateResponse(response.ResponseHeader); + Assert.That(takeoverHeader.AuthenticationToken, Is.EqualTo(originalToken)); + } + finally + { + if (backup != null) + { + await backupFixture.StopAsync(); + } + if (active != null) + { + await activeFixture.StopAsync(); + } + } + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/SharedSessionStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/SharedSessionStoreTests.cs new file mode 100644 index 0000000000..7004381361 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/SharedSessionStoreTests.cs @@ -0,0 +1,272 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class SharedSessionStoreTests + { + private const ushort NamespaceIndex = 1; + private IServiceMessageContext m_context = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:session"); + m_context = messageContext; + } + + [Test] + public async Task PutAndTryGetRoundTripsAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new SharedKeyValueSessionStore(kv, m_context); + SharedSessionEntry entry = NewEntry("tok-1"); + + await store.PutAsync(entry); + SharedSessionEntry? loaded = await store.TryGetAsync(entry.AuthenticationToken); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded!.SessionId, Is.EqualTo(entry.SessionId)); + Assert.That(loaded.AuthenticationToken, Is.EqualTo(entry.AuthenticationToken)); + Assert.That(loaded.SessionName, Is.EqualTo(entry.SessionName)); + Assert.That(loaded.CreatedAt, Is.EqualTo(entry.CreatedAt)); + Assert.That(loaded.LastActivatedAt, Is.EqualTo(entry.LastActivatedAt)); + Assert.That(loaded.SecretMaterial.ToArray(), Is.EqualTo(entry.SecretMaterial.ToArray())); + } + + [Test] + public async Task FullEntryRoundTripsAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new SharedKeyValueSessionStore(kv, m_context); + SharedSessionEntry entry = NewEntry("tok-full"); + + await store.PutAsync(entry); + SharedSessionEntry? loaded = await store.TryGetAsync(entry.AuthenticationToken); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded!.ServerNonce.ToArray(), Is.EqualTo(entry.ServerNonce.ToArray())); + Assert.That(loaded.ClientNonce.ToArray(), Is.EqualTo(entry.ClientNonce.ToArray())); + Assert.That( + loaded.ClientCertificateChain.ToArray(), + Is.EqualTo(entry.ClientCertificateChain.ToArray())); + Assert.That(loaded.SecurityPolicyUri, Is.EqualTo(entry.SecurityPolicyUri)); + Assert.That(loaded.SecurityMode, Is.EqualTo(entry.SecurityMode)); + Assert.That(loaded.EndpointUrl, Is.EqualTo(entry.EndpointUrl)); + Assert.That(loaded.SessionTimeout, Is.EqualTo(entry.SessionTimeout)); + Assert.That( + loaded.ClientDescription.ApplicationUri, + Is.EqualTo(entry.ClientDescription.ApplicationUri)); + } + + [Test] + public async Task TryGetMissingReturnsNullAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new SharedKeyValueSessionStore(kv, m_context); + + SharedSessionEntry? loaded = await store.TryGetAsync(new NodeId("nope", NamespaceIndex)); + + Assert.That(loaded, Is.Null); + } + + [Test] + public async Task RemoveDeletesEntryAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new SharedKeyValueSessionStore(kv, m_context); + SharedSessionEntry entry = NewEntry("tok-2"); + await store.PutAsync(entry); + + bool removed = await store.RemoveAsync(entry.AuthenticationToken); + SharedSessionEntry? loaded = await store.TryGetAsync(entry.AuthenticationToken); + + Assert.That(removed, Is.True); + Assert.That(loaded, Is.Null); + } + + [Test] + public async Task SessionVisibleToOtherReplicaSharingStoreAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var active = new SharedKeyValueSessionStore(kv, m_context); + var standby = new SharedKeyValueSessionStore(kv, m_context); + SharedSessionEntry entry = NewEntry("tok-3"); + + await active.PutAsync(entry); + SharedSessionEntry? onStandby = await standby.TryGetAsync(entry.AuthenticationToken); + + Assert.That(onStandby, Is.Not.Null, "standby can reconnect the session using just the token"); + Assert.That(onStandby!.SessionId, Is.EqualTo(entry.SessionId)); + } + + [Test] + public async Task ProtectedSessionRoundTripsAndIsEncryptedAtRestAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(21)); + var store = new SharedKeyValueSessionStore(kv, m_context, protector); + SharedSessionEntry entry = NewEntry("tok-prot"); + + await store.PutAsync(entry); + SharedSessionEntry? loaded = await store.TryGetAsync(entry.AuthenticationToken); + + // Secret material and the server nonce must not be persisted in cleartext. + (bool rawFound, ByteString raw) = await kv.TryGetAsync( + SharedKeyValueSessionStore.KeyFor(entry.AuthenticationToken)); + Assert.That(rawFound, Is.True); + Assert.That(Contains(raw.ToArray(), entry.SecretMaterial.ToArray()), Is.False); + Assert.That(Contains(raw.ToArray(), entry.ServerNonce.ToArray()), Is.False); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded!.SecretMaterial.ToArray(), Is.EqualTo(entry.SecretMaterial.ToArray())); + Assert.That(loaded.ServerNonce.ToArray(), Is.EqualTo(entry.ServerNonce.ToArray())); + } + + [Test] + public async Task TamperedProtectedSessionIsRejectedFailClosedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(22)); + var store = new SharedKeyValueSessionStore(kv, m_context, protector); + SharedSessionEntry entry = NewEntry("tok-tamper"); + await store.PutAsync(entry); + + await kv.SetAsync( + SharedKeyValueSessionStore.KeyFor(entry.AuthenticationToken), + ByteString.From(new byte[] { 1, 2, 3, 4, 5, 6 })); + + SharedSessionEntry? loaded = await store.TryGetAsync(entry.AuthenticationToken); + + Assert.That(loaded, Is.Null); + } + + [Test] + public async Task KeyspaceDoesNotExposeRawTokenAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var store = new SharedKeyValueSessionStore(kv, m_context); + SharedSessionEntry entry = NewEntry("tok-keyspace"); + await store.PutAsync(entry); + + string tokenText = entry.AuthenticationToken.ToString(); + await foreach (KeyValuePair pair in kv.ScanAsync("session/")) + { + Assert.That( + pair.Key.IndexOf(tokenText, StringComparison.Ordinal), + Is.LessThan(0), + "the raw authentication token must not appear in the keyspace"); + } + + // The legacy raw-token key does not exist; the hashed key resolves the entry. + (bool legacyFound, _) = await kv.TryGetAsync("session/" + tokenText); + Assert.That(legacyFound, Is.False); + Assert.That(await store.TryGetAsync(entry.AuthenticationToken), Is.Not.Null); + } + + private static bool Contains(byte[] haystack, byte[] needle) + { + for (int i = 0; i + needle.Length <= haystack.Length; i++) + { + bool match = true; + for (int j = 0; j < needle.Length; j++) + { + if (haystack[i + j] != needle[j]) + { + match = false; + break; + } + } + if (match) + { + return true; + } + } + return false; + } + + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(seed + i); + } + return key; + } + + private SharedSessionEntry NewEntry(string token) + { + return new SharedSessionEntry + { + SessionId = new NodeId(42, NamespaceIndex), + AuthenticationToken = new NodeId(token, NamespaceIndex), + SessionName = "Session " + token, + CreatedAt = DateTimeUtc.Now, + LastActivatedAt = DateTimeUtc.Now, + ServerNonce = ByteString.From(new byte[] { 40, 50, 60, 70 }), + ClientNonce = ByteString.From(new byte[] { 1, 2, 3, 4 }), + ClientCertificateChain = ByteString.From(new byte[] { 5, 6, 7, 8, 9 }), + SecurityPolicyUri = "http://opcfoundation.org/UA/SecurityPolicy#Basic256Sha256", + SecurityMode = (int)MessageSecurityMode.SignAndEncrypt, + EndpointUrl = "opc.tcp://localhost:4840", + SessionTimeout = 60000, + ClientDescription = new ApplicationDescription + { + ApplicationName = new LocalizedText("Test Client"), + ApplicationUri = "urn:test:client:" + token, + ApplicationType = ApplicationType.Client + }, + SecretMaterial = ByteString.From(new byte[] { 10, 20, 30 }) + }; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/SingleUseNonceRegistryTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/SingleUseNonceRegistryTests.cs new file mode 100644 index 0000000000..d600a85c9d --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Sessions/SingleUseNonceRegistryTests.cs @@ -0,0 +1,115 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for : a server nonce + /// can be consumed at most once across the whole replica set, so a replayed + /// ActivateSession is rejected on every replica. + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class SingleUseNonceRegistryTests + { + [Test] + public async Task FirstConsumeSucceedsSecondIsRejectedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(kv); + ByteString nonce = ByteString.From(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }); + + bool first = await registry.TryConsumeAsync(nonce); + bool second = await registry.TryConsumeAsync(nonce); + + Assert.That(first, Is.True, "first consumption is accepted"); + Assert.That(second, Is.False, "a replay of the same nonce is rejected"); + } + + [Test] + public async Task DistinctNoncesAreEachAcceptedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(kv); + + bool a = await registry.TryConsumeAsync(ByteString.From(new byte[] { 1 })); + bool b = await registry.TryConsumeAsync(ByteString.From(new byte[] { 2 })); + + Assert.That(a, Is.True); + Assert.That(b, Is.True); + } + + [Test] + public async Task NonceConsumedOnOneReplicaIsRejectedOnAnotherAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var active = new SharedSingleUseNonceRegistry(kv); + var standby = new SharedSingleUseNonceRegistry(kv); + ByteString nonce = ByteString.From(new byte[] { 9, 9, 9, 9 }); + + bool onActive = await active.TryConsumeAsync(nonce); + bool onStandby = await standby.TryConsumeAsync(nonce); + + Assert.That(onActive, Is.True); + Assert.That(onStandby, Is.False, "no two replicas accept the same nonce"); + } + + [Test] + public void NullOrEmptyNonceThrows() + { + using var kv = new InMemorySharedKeyValueStore(); + var registry = new SharedSingleUseNonceRegistry(kv); + + Assert.That( + async () => await registry.TryConsumeAsync(default), + Throws.ArgumentException); + Assert.That( + async () => await registry.TryConsumeAsync(ByteString.Empty), + Throws.ArgumentException); + } + + [Test] + public void NullStoreThrows() + { + Assert.That(() => new SharedSingleUseNonceRegistry(null!), Throws.ArgumentNullException); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/State/ByteStringCrdtSerializerTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/State/ByteStringCrdtSerializerTests.cs new file mode 100644 index 0000000000..4781cf05ae --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/State/ByteStringCrdtSerializerTests.cs @@ -0,0 +1,103 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.IO; +using System.Text.Json; +using Crdt; +using NUnit.Framework; + +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Redundancy.Server.Tests +{ + /// + /// Round-trip tests for the , which + /// must distinguish a null from an empty one. + /// + [TestFixture] + [Category("Distributed")] + public sealed class ByteStringCrdtSerializerTests + { + [Test] + public void RoundTripsNullEmptyAndDataValues() + { + var clock = new HybridLogicalClock(ReplicaId.FromUInt64(1), TimeProvider.System); + var map = new LWWMap(); + map.Set("data", new ByteString(new byte[] { 9, 8, 7 }), clock); + map.Set("empty", new ByteString(Array.Empty()), clock); + map.Set("null", default, clock); + + byte[] bytes = map.ToByteArray(CrdtValues.String, ByteStringCrdtSerializer.Instance); + LWWMap restored = LWWMap.ReadFrom( + bytes, CrdtValues.String, ByteStringCrdtSerializer.Instance, CrdtReaderOptions.Default); + + Assert.That(restored.TryGetValue("data", out ByteString data), Is.True); + Assert.That(data.ToArray(), Is.EqualTo(new byte[] { 9, 8, 7 })); + + Assert.That(restored.TryGetValue("empty", out ByteString empty), Is.True); + Assert.That(empty.IsNull, Is.False, "an empty ByteString must not decode as null"); + Assert.That(empty.ToArray(), Is.Empty); + + Assert.That(restored.TryGetValue("null", out ByteString nul), Is.True); + Assert.That(nul.IsNull, Is.True, "a null ByteString must decode as null"); + } + + [Test] + public void RoundTripsJsonNullAndDataValues() + { + ByteString data = JsonRoundTrip(new ByteString(new byte[] { 4, 5, 6 })); + Assert.That(data.IsNull, Is.False); + Assert.That(data.ToArray(), Is.EqualTo(new byte[] { 4, 5, 6 })); + + ByteString empty = JsonRoundTrip(new ByteString(Array.Empty())); + Assert.That(empty.IsNull, Is.False, "an empty ByteString must not decode as null"); + Assert.That(empty.ToArray(), Is.Empty); + + ByteString nul = JsonRoundTrip(default); + Assert.That(nul.IsNull, Is.True, "a null ByteString must decode as null"); + } + + private static ByteString JsonRoundTrip(ByteString value) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + ByteStringCrdtSerializer.Instance.WriteJson(writer, value); + writer.Flush(); + } + + var reader = new Utf8JsonReader(stream.ToArray()); + reader.Read(); + return ByteStringCrdtSerializer.Instance.ReadJson(ref reader); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/State/CrdtAddressSpaceStartupTaskTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/State/CrdtAddressSpaceStartupTaskTests.cs new file mode 100644 index 0000000000..3a3318b751 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/State/CrdtAddressSpaceStartupTaskTests.cs @@ -0,0 +1,119 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; + +using Opc.Ua.Server; + +namespace Opc.Ua.Redundancy.Server.Tests +{ + /// + /// Tests for : it attaches a CRDT + /// synchronizer to every opted-in node manager and skips the rest. + /// + [TestFixture] + [Category("Distributed")] + public sealed class CrdtAddressSpaceStartupTaskTests + { + private const ushort NamespaceIndex = 1; + + [Test] + public async Task AttachesSynchronizerToOptedInNodeManagerAsync() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:crdt-startup"); + var systemContext = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + + var addressSpace = new DictionaryAddressSpace(systemContext); + await addressSpace.AddOrUpdateNodeAsync(new BaseDataVariableState(null) + { + NodeId = new NodeId("seeded", NamespaceIndex), + BrowseName = new QualifiedName("Seeded", NamespaceIndex), + DisplayName = new LocalizedText("Seeded"), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(1.0) + }); + + var optedIn = new Mock(); + optedIn.As() + .Setup(s => s.CreateLocalAddressSpace()) + .Returns(addressSpace); + var notOptedIn = new Mock(); + + var masterNodeManager = new Mock(); + masterNodeManager.Setup(m => m.NodeManagers) + .Returns(new[] { optedIn.Object, notOptedIn.Object }); + + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(telemetry); + server.Setup(s => s.MessageContext).Returns(messageContext); + server.Setup(s => s.NodeManager).Returns(masterNodeManager.Object); + + await using var task = new CrdtAddressSpaceStartupTask( + EmptyServices(), new ReplicatedAddressSpaceOptions()); + + await task.OnServerStartedAsync(server.Object); + + // The opted-in manager exposed its local address space. + optedIn.As().Verify(s => s.CreateLocalAddressSpace(), Times.Once); + } + + [Test] + public void ConstructorRejectsNullArguments() + { + Assert.That( + () => new CrdtAddressSpaceStartupTask(null!, new ReplicatedAddressSpaceOptions()), + Throws.ArgumentNullException); + Assert.That( + () => new CrdtAddressSpaceStartupTask(EmptyServices(), null!), + Throws.ArgumentNullException); + } + + private static IServiceProvider EmptyServices() + { + return Mock.Of(); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/State/CrdtAddressSpaceSynchronizerTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/State/CrdtAddressSpaceSynchronizerTests.cs new file mode 100644 index 0000000000..83c40570d3 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/State/CrdtAddressSpaceSynchronizerTests.cs @@ -0,0 +1,249 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Crdt; +using Crdt.Transport; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Redundancy.Server.Tests +{ + /// + /// Active/active convergence tests for + /// running two replicas over a deterministic in-memory gossip network. + /// + [TestFixture] + [Category("Distributed")] + public sealed class CrdtAddressSpaceSynchronizerTests + { + private const ushort NamespaceIndex = 1; + private IServiceMessageContext m_messageContext = null!; + private SystemContext m_systemContext = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:crdt"); + m_messageContext = messageContext; + m_systemContext = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + } + + [Test] + public async Task ReplicatesAddedNodeToPeerAsync() + { + await using var fixture = await TwoReplicaFixture.CreateAsync(this); + + BaseDataVariableState node = NewVariable("added", 1.0); + await fixture.SpaceA.AddOrUpdateNodeAsync(node); + + await AssertEventuallyAsync( + () => fixture.SpaceB.TryGetNode(node.NodeId, out _), + "node added on A should replicate to B"); + } + + [Test] + public async Task ReplicatesValueUpdateAsync() + { + await using var fixture = await TwoReplicaFixture.CreateAsync(this); + + BaseDataVariableState node = NewVariable("value", 1.0); + await fixture.SpaceA.AddOrUpdateNodeAsync(node); + await AssertEventuallyAsync( + () => fixture.SpaceB.TryGetNode(node.NodeId, out _), + "node should replicate before the value update"); + + node.Value = new Variant(42.0); + node.ClearChangeMasks(m_systemContext, false); + + await AssertEventuallyAsync( + () => fixture.SpaceB.TryGetNode(node.NodeId, out NodeState? remote) && + remote is BaseVariableState variable && + variable.Value.Equals(new Variant(42.0)), + "value updated on A should replicate to B"); + } + + [Test] + public async Task ReplicatesRemovalAsync() + { + await using var fixture = await TwoReplicaFixture.CreateAsync(this); + + BaseDataVariableState node = NewVariable("removed", 1.0); + await fixture.SpaceA.AddOrUpdateNodeAsync(node); + await AssertEventuallyAsync( + () => fixture.SpaceB.TryGetNode(node.NodeId, out _), + "node should replicate before removal"); + + await fixture.SpaceA.RemoveNodeAsync(node.NodeId); + + await AssertEventuallyAsync( + () => !fixture.SpaceB.TryGetNode(node.NodeId, out _), + "removal on A should replicate to B"); + } + + [Test] + public async Task MultiWriterConvergesBothDirectionsAsync() + { + await using var fixture = await TwoReplicaFixture.CreateAsync(this); + + BaseDataVariableState onA = NewVariable("from-a", 1.0); + BaseDataVariableState onB = NewVariable("from-b", 2.0); + await fixture.SpaceA.AddOrUpdateNodeAsync(onA); + await fixture.SpaceB.AddOrUpdateNodeAsync(onB); + + await AssertEventuallyAsync( + () => fixture.SpaceA.TryGetNode(onB.NodeId, out _) && + fixture.SpaceB.TryGetNode(onA.NodeId, out _), + "each replica's write should converge on the other (no leader)"); + } + + [Test] + public async Task ConcurrentValueWritesConvergeAsync() + { + await using var fixture = await TwoReplicaFixture.CreateAsync(this); + + BaseDataVariableState seed = NewVariable("contended", 0.0); + await fixture.SpaceA.AddOrUpdateNodeAsync(seed); + await AssertEventuallyAsync( + () => fixture.SpaceB.TryGetNode(seed.NodeId, out _), + "node should replicate before concurrent writes"); + + // Concurrent writes on both replicas; last-writer-wins must converge + // both sides to the same value. + seed.Value = new Variant(11.0); + seed.ClearChangeMasks(m_systemContext, false); + if (fixture.SpaceB.TryGetNode(seed.NodeId, out NodeState? onB) && onB is BaseVariableState bVar) + { + bVar.Value = new Variant(22.0); + bVar.ClearChangeMasks(m_systemContext, false); + } + + await AssertEventuallyAsync( + () => fixture.SpaceA.TryGetNode(seed.NodeId, out NodeState? a) && + fixture.SpaceB.TryGetNode(seed.NodeId, out NodeState? b) && + a is BaseVariableState av && b is BaseVariableState bv && + av.Value.Equals(bv.Value), + "concurrent value writes must converge to the same value on both replicas"); + } + + private BaseDataVariableState NewVariable(string id, double value) + { + return new BaseDataVariableState(null) + { + NodeId = new NodeId(id, NamespaceIndex), + BrowseName = new QualifiedName(id, NamespaceIndex), + DisplayName = new LocalizedText(id), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(value) + }; + } + + private static async Task AssertEventuallyAsync(Func condition, string message) + { + // CRDT convergence over the in-memory gossip network is normally sub-second, + // but the background capture/broadcast loops can be starved of CPU on a heavily + // loaded CI runner (the full test matrix runs dozens of jobs per runner). Allow a + // generous deadline so the assertion measures convergence correctness, not runner load. + DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + await Task.Delay(25); + } + Assert.Fail(message); + } + + private sealed class TwoReplicaFixture : IAsyncDisposable + { + public DictionaryAddressSpace SpaceA { get; private set; } = null!; + public DictionaryAddressSpace SpaceB { get; private set; } = null!; + + public static async Task CreateAsync(CrdtAddressSpaceSynchronizerTests test) + { + var fixture = new TwoReplicaFixture(); + fixture.m_network = new InMemoryNetwork(); + fixture.SpaceA = new DictionaryAddressSpace(test.m_systemContext); + fixture.SpaceB = new DictionaryAddressSpace(test.m_systemContext); + + fixture.m_syncA = new CrdtAddressSpaceSynchronizer( + fixture.SpaceA, test.m_messageContext, ReplicaId.FromUInt64(1), + fixture.m_network.CreateTransport(), TimeProvider.System, CrdtReaderOptions.Default); + fixture.m_syncB = new CrdtAddressSpaceSynchronizer( + fixture.SpaceB, test.m_messageContext, ReplicaId.FromUInt64(2), + fixture.m_network.CreateTransport(), TimeProvider.System, CrdtReaderOptions.Default); + + // Start both transports first so neither misses the other's + // broadcasts, then begin capturing/applying changes. + await fixture.m_syncA.SeedOrHydrateAsync(); + await fixture.m_syncB.SeedOrHydrateAsync(); + fixture.m_syncA.Start(); + fixture.m_syncB.Start(); + return fixture; + } + + public async ValueTask DisposeAsync() + { + if (m_syncA != null) + { + await m_syncA.DisposeAsync(); + } + if (m_syncB != null) + { + await m_syncB.DisposeAsync(); + } + if (m_network != null) + { + await m_network.DisposeAsync(); + } + } + + private InMemoryNetwork? m_network; + private CrdtAddressSpaceSynchronizer? m_syncA; + private CrdtAddressSpaceSynchronizer? m_syncB; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/DeterministicEventIdProviderTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/DeterministicEventIdProviderTests.cs new file mode 100644 index 0000000000..ec3b0bf951 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/DeterministicEventIdProviderTests.cs @@ -0,0 +1,241 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using System.Reflection; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.Fluent; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Category("Events")] + public class DeterministicEventIdProviderTests + { + [Test] + public void SameEventContentProducesSameEventIdAcrossReplicas() + { + var providerA = new DeterministicEventIdProvider("replica-set"); + var providerB = new DeterministicEventIdProvider("replica-set"); + ServerSystemContext context = CreateContext(); + BaseObjectState notifier = CreateNotifier(); + BaseEventState eventA = CreateEvent("started"); + BaseEventState eventB = CreateEvent("started"); + eventA.ReceiveTime!.Value = new DateTimeUtc(638000000000000001); + eventB.ReceiveTime!.Value = new DateTimeUtc(638000000000000999); + + ByteString idA = providerA.CreateEventId(notifier, context, eventA); + ByteString idB = providerB.CreateEventId(notifier, context, eventB); + + Assert.That(idA.ToArray(), Is.EqualTo(idB.ToArray())); + Assert.That(idA.Length, Is.EqualTo(32)); + } + + [Test] + public void DifferentSeedProducesDifferentEventId() + { + ServerSystemContext context = CreateContext(); + BaseObjectState notifier = CreateNotifier(); + BaseEventState e = CreateEvent("started"); + + ByteString idA = new DeterministicEventIdProvider("set-a").CreateEventId(notifier, context, e); + ByteString idB = new DeterministicEventIdProvider("set-b").CreateEventId(notifier, context, e); + + Assert.That(idA.ToArray(), Is.Not.EqualTo(idB.ToArray())); + } + + [Test] + public void ConstructorRejectsEmptySeed() + { + Assert.That( + () => new DeterministicEventIdProvider(" "), + Throws.ArgumentException); + } + + [Test] + public void DifferentEventTimeProducesDifferentEventId() + { + var provider = new DeterministicEventIdProvider("replica-set"); + ServerSystemContext context = CreateContext(); + BaseObjectState notifier = CreateNotifier(); + BaseEventState event1 = CreateEvent("started"); + event1.Time!.Value = new DateTimeUtc(638000000000000000); + BaseEventState event2 = CreateEvent("started"); + event2.Time!.Value = new DateTimeUtc(639000000000000000); + + ByteString id1 = provider.CreateEventId(notifier, context, event1); + ByteString id2 = provider.CreateEventId(notifier, context, event2); + + Assert.That(id1.ToArray(), Is.Not.EqualTo(id2.ToArray())); + } + + [Test] + public void DistinctEventMessagesProduceDifferentEventIds() + { + var provider = new DeterministicEventIdProvider("replica-set"); + ServerSystemContext context = CreateContext(); + BaseObjectState notifier = CreateNotifier(); + BaseEventState event1 = CreateEvent("started"); + BaseEventState event2 = CreateEvent("stopped"); + + ByteString id1 = provider.CreateEventId(notifier, context, event1); + ByteString id2 = provider.CreateEventId(notifier, context, event2); + + Assert.That(id1.ToArray(), Is.Not.EqualTo(id2.ToArray())); + } + + [Test] + public void EventSourceRegistryComputesEventIdAfterDefaultDistinguishingFields() + { + var provider = new CapturingEventIdProvider(); + ServerSystemContext context = CreateContext(); + BaseObjectState notifier = CreateNotifier(); + var e = new BaseEventState(null); + + InvokePopulateDefaults(notifier, context, e, provider); + + Assert.That(provider.ObservedMessage, Is.Not.Null); + Assert.That(provider.ObservedSeverity, Is.EqualTo((ushort)EventSeverity.Medium)); + Assert.That(e.EventId, Is.Not.Null); + Assert.That(e.EventId!.Value.ToArray(), Is.EqualTo(new byte[] { 1, 2, 3, 4 })); + } + + [Test] + public void DifferentNotifierProducesDifferentEventId() + { + var provider = new DeterministicEventIdProvider("replica-set"); + ServerSystemContext context = CreateContext(); + BaseObjectState notifier1 = CreateNotifier(); + notifier1.NodeId = new NodeId("Notifier1", 2); + BaseObjectState notifier2 = CreateNotifier(); + notifier2.NodeId = new NodeId("Notifier2", 2); + BaseEventState e = CreateEvent("started"); + + ByteString id1 = provider.CreateEventId(notifier1, context, e); + ByteString id2 = provider.CreateEventId(notifier2, context, e); + + Assert.That(id1.ToArray(), Is.Not.EqualTo(id2.ToArray())); + } + + [Test] + public void EventIdLengthIs32Bytes() + { + var provider = new DeterministicEventIdProvider("replica-set"); + ServerSystemContext context = CreateContext(); + BaseObjectState notifier = CreateNotifier(); + BaseEventState e = CreateEvent("started"); + + ByteString id = provider.CreateEventId(notifier, context, e); + + Assert.That(id.Length, Is.EqualTo(32)); + } + + [Test] + public void ConstructorRejectsNullSeed() + { + Assert.That( + () => new DeterministicEventIdProvider(null!), + Throws.ArgumentException); + } + + private static ServerSystemContext CreateContext() + { + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(NUnitTelemetryContext.Create()); + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(NUnitTelemetryContext.Create()); + server.Setup(s => s.NamespaceUris).Returns(messageContext.NamespaceUris); + server.Setup(s => s.ServerUris).Returns(messageContext.ServerUris); + server.Setup(s => s.Factory).Returns(messageContext.Factory); + server.Setup(s => s.TypeTree).Returns(new TypeTable(messageContext.NamespaceUris)); + return new ServerSystemContext(server.Object); + } + + private static BaseObjectState CreateNotifier() + { + return new BaseObjectState(null) + { + NodeId = new NodeId("Notifier", 2), + BrowseName = new QualifiedName("Notifier", 2) + }; + } + + private static BaseEventState CreateEvent(string message) + { + var e = new BaseEventState(null) + { + Message = PropertyState.With(null!, new LocalizedText(message)), + Severity = PropertyState.With(null!, 500), + Time = PropertyState.With(null!, new DateTimeUtc(638000000000000000)), + ReceiveTime = PropertyState.With( + null!, + new DateTimeUtc(638000000000000000)) + }; + return e; + } + + private static void InvokePopulateDefaults( + BaseObjectState notifier, + ISystemContext context, + BaseEventState e, + IEventIdProvider provider) + { + Type registryType = typeof(IEventIdProvider).Assembly.GetType( + "Opc.Ua.Server.Fluent.EventSourceRegistry")!; + MethodInfo method = registryType.GetMethod( + "PopulateDefaults", + BindingFlags.Static | BindingFlags.NonPublic)!; + method.Invoke(null, new object[] { notifier, context, e, provider }); + } + + private sealed class CapturingEventIdProvider : IEventIdProvider + { + public LocalizedText? ObservedMessage { get; private set; } + + public ushort? ObservedSeverity { get; private set; } + + public ByteString CreateEventId(BaseObjectState notifier, ISystemContext context, BaseEventState eventState) + { + _ = notifier; + _ = context; + ObservedMessage = eventState.Message?.Value; + ObservedSeverity = eventState.Severity?.Value; + return ByteString.From(new byte[] { 1, 2, 3, 4 }); + } + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/RegisteredNodesMirrorTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/RegisteredNodesMirrorTests.cs new file mode 100644 index 0000000000..d15abc033d --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/RegisteredNodesMirrorTests.cs @@ -0,0 +1,89 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +#nullable enable + +using System; +using Moq; +using NUnit.Framework; +using Opc.Ua.Tests; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Verifies that registered node ids are already replica-stable. + /// + [TestFixture] + [Category("Distributed")] + [Category("RegisteredNodes")] + public class RegisteredNodesMirrorTests + { + [Test] + public void RegisterNodesReturnsInputNodeIdsWithoutReplicaLocalAliases() + { + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(NUnitTelemetryContext.Create()); + server.Setup(s => s.NamespaceUris).Returns(new NamespaceTable()); + var nodeManagerFactory = new Mock(); + var configurationNodeManager = new Mock(); + configurationNodeManager.Setup(n => n.NamespaceUris).Returns(Array.Empty()); + var coreNodeManager = new Mock(); + nodeManagerFactory + .Setup(f => f.CreateConfigurationNodeManager()) + .Returns(configurationNodeManager.Object); + nodeManagerFactory + .Setup(f => f.CreateCoreNodeManager(It.IsAny())) + .Returns(coreNodeManager.Object); + server.Setup(s => s.MainNodeManagerFactory).Returns(nodeManagerFactory.Object); + var manager = new MasterNodeManager( + server.Object, + new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration + { + MaxBrowseContinuationPoints = 10 + } + }, + null, + Array.Empty()); + ArrayOf input = + [ + new NodeId("Temperature", 2), + ObjectIds.Server + ]; + + manager.RegisterNodes( + new OperationContext(new Mock().Object, DiagnosticsMasks.None), + input, + out ArrayOf registered); + + Assert.That(registered, Is.EqualTo(input)); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/SharedContinuationPointStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/SharedContinuationPointStoreTests.cs new file mode 100644 index 0000000000..81cc937958 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/SharedContinuationPointStoreTests.cs @@ -0,0 +1,296 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Tests best-effort continuation point envelope mirroring. + /// + [TestFixture] + [Category("Distributed")] + [Category("ContinuationPoints")] + public class SharedContinuationPointStoreTests + { + [Test] + public async Task BrowseContinuationPointEnvelopeReplicatesAcrossStoresAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + Guid continuationPointId = Guid.NewGuid(); + + primary.StoreContinuationPoint(CreateBrowseEnvelope(sessionId, continuationPointId)); + await primary.FlushAsync(); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Has.Count.EqualTo(1)); + ContinuationPointEnvelope envelope = envelopes[0]; + Assert.That(envelope.Id, Is.EqualTo(continuationPointId)); + Assert.That(envelope.OwnerSessionId, Is.EqualTo(sessionId)); + Assert.That(envelope.Kind, Is.EqualTo(ContinuationPointKind.Browse)); + Assert.That(envelope.BrowseNodeId, Is.EqualTo(new NodeId("Demo", 2))); + Assert.That(envelope.MaxResultsToReturn, Is.EqualTo(10)); + Assert.That(envelope.ResultMask, Is.EqualTo(BrowseResultMask.All)); + } + + [Test] + public async Task RemovedContinuationPointEnvelopeDoesNotLoadOnBackupAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + Guid continuationPointId = Guid.NewGuid(); + + primary.StoreContinuationPoint(CreateBrowseEnvelope(sessionId, continuationPointId)); + await primary.FlushAsync(); + primary.RemoveContinuationPoint(sessionId, ContinuationPointKind.Browse, continuationPointId); + await primary.FlushAsync(); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Is.Empty); + } + + [Test] + public async Task BrowseNextForMirroredOpaqueContinuationPointFailsGracefullyAsync() + { + MasterNodeManager manager = CreateMasterNodeManager(); + var session = new Mock(); + ByteString continuationPoint = Guid.NewGuid().ToByteArray().ToByteString(); + session + .Setup(s => s.RestoreContinuationPoint(continuationPoint)) + .Returns((ContinuationPoint?)null); + var context = new OperationContext(session.Object, DiagnosticsMasks.None); + + (ArrayOf results, _) = await manager.BrowseNextAsync( + context, + false, + [continuationPoint]); + + Assert.That(results, Has.Count.EqualTo(1)); + Assert.That(results[0].StatusCode, Is.EqualTo(StatusCodes.BadContinuationPointInvalid)); + } + + [Test] + public async Task LoadUnknownSessionReturnsNoEnvelopesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var store = CreateStore(kv); + + ArrayOf envelopes = + await store.LoadContinuationPointsAsync(new NodeId(Guid.NewGuid(), 1)); + + Assert.That(envelopes, Is.Empty); + } + + [Test] + public async Task SecondBrowseContinuationPointEnvelopeReplicatesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + Guid continuationPointId = Guid.NewGuid(); + + primary.StoreContinuationPoint(new ContinuationPointEnvelope + { + Id = continuationPointId, + OwnerSessionId = sessionId, + Kind = ContinuationPointKind.Browse, + Index = 5 + }); + await primary.FlushAsync(); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Has.Count.EqualTo(1)); + Assert.That(envelopes[0].Kind, Is.EqualTo(ContinuationPointKind.Browse)); + Assert.That(envelopes[0].Index, Is.EqualTo(5)); + } + + [Test] + public async Task HistoryContinuationPointEnvelopeReplicatesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + Guid continuationPointId = Guid.NewGuid(); + + primary.StoreContinuationPoint(new ContinuationPointEnvelope + { + Id = continuationPointId, + OwnerSessionId = sessionId, + Kind = ContinuationPointKind.History, + BrowseNodeId = new NodeId("HistoryVar", 2) + }); + await primary.FlushAsync(); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Has.Count.EqualTo(1)); + Assert.That(envelopes[0].Kind, Is.EqualTo(ContinuationPointKind.History)); + Assert.That(envelopes[0].BrowseNodeId, Is.EqualTo(new NodeId("HistoryVar", 2))); + } + + [Test] + public async Task MultipleContinuationPointsForSameSessionReplicateAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + + primary.StoreContinuationPoint(CreateBrowseEnvelope(sessionId, Guid.NewGuid())); + primary.StoreContinuationPoint(new ContinuationPointEnvelope + { + Id = Guid.NewGuid(), + OwnerSessionId = sessionId, + Kind = ContinuationPointKind.History + }); + await primary.FlushAsync(); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Has.Count.EqualTo(2)); + } + + [Test] + public async Task ContinuationPointForDifferentSessionDoesNotLoadAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionA = new NodeId(Guid.NewGuid(), 1); + NodeId sessionB = new NodeId(Guid.NewGuid(), 1); + + primary.StoreContinuationPoint(CreateBrowseEnvelope(sessionA, Guid.NewGuid())); + await primary.FlushAsync(); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionB); + + Assert.That(envelopes, Is.Empty); + } + + [Test] + public async Task FlushBeforeStoreCausesNoReplicationAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var primary = CreateStore(kv); + await using var backup = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + + await primary.FlushAsync(); + primary.StoreContinuationPoint(CreateBrowseEnvelope(sessionId, Guid.NewGuid())); + + ArrayOf envelopes = + await backup.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Is.Empty); + } + + private static SharedKeyValueSubscriptionStore CreateStore(InMemorySharedKeyValueStore kv) + { + return new SharedKeyValueSubscriptionStore( + kv, + ServiceMessageContext.CreateEmpty(NUnitTelemetryContext.Create())); + } + + private static ContinuationPointEnvelope CreateBrowseEnvelope(NodeId sessionId, Guid id) + { + return new ContinuationPointEnvelope + { + Id = id, + OwnerSessionId = sessionId, + Kind = ContinuationPointKind.Browse, + BrowseNodeId = new NodeId("Demo", 2), + View = new ViewDescription { ViewId = ObjectIds.ViewsFolder }, + MaxResultsToReturn = 10, + BrowseDirection = BrowseDirection.Forward, + ReferenceTypeId = ReferenceTypeIds.HierarchicalReferences, + IncludeSubtypes = true, + NodeClassMask = (uint)NodeClass.Variable, + ResultMask = BrowseResultMask.All, + Index = 3 + }; + } + + private static MasterNodeManager CreateMasterNodeManager() + { + var server = new Mock(); + server.Setup(s => s.Telemetry).Returns(NUnitTelemetryContext.Create()); + server.Setup(s => s.NamespaceUris).Returns(new NamespaceTable()); + var nodeManagerFactory = new Mock(); + var configurationNodeManager = new Mock(); + configurationNodeManager.Setup(n => n.NamespaceUris).Returns(Array.Empty()); + var coreNodeManager = new Mock(); + nodeManagerFactory + .Setup(f => f.CreateConfigurationNodeManager()) + .Returns(configurationNodeManager.Object); + nodeManagerFactory + .Setup(f => f.CreateCoreNodeManager(It.IsAny())) + .Returns(coreNodeManager.Object); + server.Setup(s => s.MainNodeManagerFactory).Returns(nodeManagerFactory.Object); + return new MasterNodeManager( + server.Object, + new ApplicationConfiguration + { + ServerConfiguration = new ServerConfiguration + { + MaxBrowseContinuationPoints = 10 + } + }, + null, + Array.Empty()); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/SharedKeyValueSubscriptionStoreTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/SharedKeyValueSubscriptionStoreTests.cs new file mode 100644 index 0000000000..41ed1eaaf2 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Subscriptions/SharedKeyValueSubscriptionStoreTests.cs @@ -0,0 +1,828 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Category("Subscription")] + [Parallelizable(ParallelScope.All)] + public class SharedKeyValueSubscriptionStoreTests + { + private const ushort NamespaceIndex = 2; + + [Test] + public async Task StoreAndRestoreRoundTripsDefinitionAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + StoredSubscription expected = NewSubscription(100, 10); + + bool stored = await store.StoreSubscriptionsAsync([expected]); + RestoreSubscriptionResult result = await store.RestoreSubscriptionsAsync(); + + Assert.That(stored, Is.True); + Assert.That(result.Success, Is.True); + StoredSubscription actual = (StoredSubscription)result.Subscriptions!.Single(); + AssertSubscription(actual, expected); + AssertMonitoredItem((StoredMonitoredItem)actual.MonitoredItems.Single(), NewItem(100, 10)); + } + + [Test] + public async Task StoreIsVisibleToSecondReplicaAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore active = CreateStore(kv); + SharedKeyValueSubscriptionStore backup = CreateStore(kv); + StoredSubscription expected = NewSubscription(200, 20); + + await active.StoreSubscriptionsAsync([expected]); + RestoreSubscriptionResult result = await backup.RestoreSubscriptionsAsync(); + + Assert.That(result.Success, Is.True); + StoredSubscription actual = (StoredSubscription)result.Subscriptions!.Single(); + AssertSubscription(actual, expected); + } + + [Test] + public async Task ProtectedDefinitionCacheSurvivesTamperedBackendRecordAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(11)); + var store = new SharedKeyValueSubscriptionStore(kv, CreateContext(), protector); + StoredSubscription subscription = NewSubscription(300, 30); + await store.StoreSubscriptionsAsync([subscription]); + + await kv.SetAsync( + SharedKeyValueSubscriptionStore.KeyFor(subscription.Id), + ByteString.From(new byte[] { 1, 2, 3, 4, 5 })); + RestoreSubscriptionResult result = await store.RestoreSubscriptionsAsync(); + + Assert.That(result.Success, Is.True); + Assert.That(result.Subscriptions, Is.Not.Empty); + } + + [Test] + public async Task ProtectedStoreDoesNotPersistDefinitionInClearTextAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(12)); + var store = new SharedKeyValueSubscriptionStore(kv, CreateContext(), protector); + StoredSubscription subscription = NewSubscription(400, 40); + + await store.StoreSubscriptionsAsync([subscription]); + (bool found, ByteString raw) = await kv.TryGetAsync(SharedKeyValueSubscriptionStore.KeyFor(subscription.Id)); + + Assert.That(found, Is.True); + Assert.That(Contains(raw.ToArray(), BitConverter.GetBytes(subscription.Id)), Is.False); + Assert.That((await store.RestoreSubscriptionsAsync()).Subscriptions!.Single().Id, Is.EqualTo(subscription.Id)); + } + + [Test] + public async Task StoreReplacesRemovedSubscriptionsAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + + await store.StoreSubscriptionsAsync([NewSubscription(500, 50), NewSubscription(501, 51)]); + await store.StoreSubscriptionsAsync([NewSubscription(501, 51)]); + RestoreSubscriptionResult result = await store.RestoreSubscriptionsAsync(); + + Assert.That(result.Subscriptions!.Select(s => s.Id), Is.EqualTo(new uint[] { 501 })); + } + + [Test] + public async Task OnSubscriptionRestoreCompleteCleansStaleDefinitionsAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + await store.StoreSubscriptionsAsync([NewSubscription(600, 60), NewSubscription(601, 61)]); + + await store.OnSubscriptionRestoreCompleteAsync(new Dictionary> + { + [601] = new ArrayOf(new uint[] { 61 }) + }); + RestoreSubscriptionResult result = await store.RestoreSubscriptionsAsync(); + + Assert.That(result.Subscriptions!.Select(s => s.Id), Is.EqualTo(new uint[] { 601 })); + } + + [Test] + public async Task RetransmissionStateIsVisibleToSecondReplicaAndKeepsSequenceContinuityAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore primary = CreateStore(kv); + SharedKeyValueSubscriptionStore backup = CreateStore(kv); + + primary.StoreRetransmissionState( + 700, + 4, + [NewNotification(1), NewNotification(2), NewNotification(3)]); + await primary.FlushAsync(); + SubscriptionRetransmissionState? state = await backup.LoadRetransmissionStateAsync(700); + + Assert.That(state, Is.Not.Null); + Assert.That(state!.NextSequenceNumber, Is.EqualTo(4)); + Assert.That( + state.SentMessages.Memory.ToArray().Select(m => m.SequenceNumber), + Is.EqualTo(new uint[] { 1, 2, 3 })); + + NotificationMessage republished = state.SentMessages.Memory.ToArray().Single(m => m.SequenceNumber == 2); + Assert.That(republished.NotificationData, Has.Count.EqualTo(1)); + } + + [Test] + public async Task RetransmissionAcknowledgeEvictsMirroredNotificationAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore primary = CreateStore(kv); + SharedKeyValueSubscriptionStore backup = CreateStore(kv); + primary.StoreRetransmissionState( + 701, + 4, + [NewNotification(1), NewNotification(2), NewNotification(3)]); + await primary.FlushAsync(); + + primary.AcknowledgeNotification(701, 2); + await primary.FlushAsync(); + SubscriptionRetransmissionState? state = await backup.LoadRetransmissionStateAsync(701); + + Assert.That(state, Is.Not.Null); + Assert.That( + state!.SentMessages.Memory.ToArray().Select(m => m.SequenceNumber), + Is.EqualTo(new uint[] { 1, 3 })); + } + + [Test] + public async Task RetransmissionSnapshotRemovesStaleSequenceKeysAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + store.StoreRetransmissionState( + 702, + 4, + [NewNotification(1), NewNotification(2), NewNotification(3)]); + await store.FlushAsync(); + + store.StoreRetransmissionState(702, 5, [NewNotification(3), NewNotification(4)]); + await store.FlushAsync(); + SubscriptionRetransmissionState? state = await store.LoadRetransmissionStateAsync(702); + + Assert.That(state, Is.Not.Null); + Assert.That(state!.NextSequenceNumber, Is.EqualTo(5)); + Assert.That( + state.SentMessages.Memory.ToArray().Select(m => m.SequenceNumber), + Is.EqualTo(new uint[] { 3, 4 })); + } + + [Test] + public async Task RetransmissionDeltaAddsAndRemovesOnlyChangedMessagesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + SharedKeyValueSubscriptionStore backup = CreateStore(kv); + + store.StoreRetransmissionStateDelta(708, 4, [NewNotification(1), NewNotification(2)], []); + await store.FlushAsync(); + store.StoreRetransmissionStateDelta(708, 5, [NewNotification(3)], [1]); + await store.FlushAsync(); + SubscriptionRetransmissionState? state = await backup.LoadRetransmissionStateAsync(708); + + Assert.That(state, Is.Not.Null); + Assert.That(state!.NextSequenceNumber, Is.EqualTo(5)); + Assert.That( + state.SentMessages.Memory.ToArray().Select(m => m.SequenceNumber), + Is.EqualTo(new uint[] { 2, 3 })); + } + + [Test] + public async Task RetransmissionMessagesDoNotRepeatNamespaceTablesAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + + store.StoreRetransmissionStateDelta(709, 2, [NewNotification(1)], []); + await store.FlushAsync(); + (bool stateFound, ByteString stateRaw) = await kv.TryGetAsync( + SharedKeyValueSubscriptionStore.RetransmissionStateKeyFor(709)); + (bool messageFound, ByteString messageRaw) = await kv.TryGetAsync( + SharedKeyValueSubscriptionStore.RetransmissionMessageKeyFor(709, 1)); + + byte[] namespaceBytes = Encoding.UTF8.GetBytes("urn:test:subscriptions"); + Assert.That(stateFound, Is.True); + Assert.That(messageFound, Is.True); + Assert.That(Contains(stateRaw.ToArray(), namespaceBytes), Is.True); + Assert.That(Contains(messageRaw.ToArray(), namespaceBytes), Is.False); + } + + [Test] + public async Task RetransmissionTamperFailsClosedAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey(13)); + var store = new SharedKeyValueSubscriptionStore(kv, CreateContext(), protector); + store.StoreRetransmissionState(703, 2, [NewNotification(1)]); + await store.FlushAsync(); + + await kv.SetAsync( + SharedKeyValueSubscriptionStore.RetransmissionStateKeyFor(703), + ByteString.From(new byte[] { 9, 8, 7 })); + SubscriptionRetransmissionState? state = await store.LoadRetransmissionStateAsync(703); + + Assert.That(state, Is.Null); + } + + [Test] + public async Task MissingRetransmissionStateReturnsNullAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + + SubscriptionRetransmissionState? state = await store.LoadRetransmissionStateAsync(704); + + Assert.That(state, Is.Null); + } + + [Test] + public async Task RetransmissionMirrorUsesAsyncBackendWithoutBlockingAsync() + { + using var inner = new InMemorySharedKeyValueStore(); + var kv = new AsyncSharedKeyValueStore(inner); + var primary = new SharedKeyValueSubscriptionStore(kv, CreateContext()); + var backup = new SharedKeyValueSubscriptionStore(kv, CreateContext()); + + primary.StoreRetransmissionState(705, 3, [NewNotification(1), NewNotification(2)]); + await primary.FlushAsync(); + SubscriptionRetransmissionState? state = await backup.LoadRetransmissionStateAsync(705); + + Assert.That(state, Is.Not.Null); + Assert.That(state!.NextSequenceNumber, Is.EqualTo(3)); + Assert.That( + state.SentMessages.Memory.ToArray().Select(m => m.SequenceNumber), + Is.EqualTo(new uint[] { 1, 2 })); + } + + [Test] + public async Task RetransmissionDeleteIsOrderedAfterInflightMirrorWritesAsync() + { + const uint subscriptionId = 706; + using var inner = new InMemorySharedKeyValueStore(); + var kv = new BlockingRetransmissionSetStore(inner, subscriptionId); + await using var store = new SharedKeyValueSubscriptionStore(kv, CreateContext()); + await store.StoreSubscriptionsAsync([NewSubscription(subscriptionId, 76)]); + + store.StoreRetransmissionState(subscriptionId, 3, [NewNotification(1), NewNotification(2)]); + await kv.WaitForBlockedSetAsync(); + await store.StoreSubscriptionsAsync([]); + kv.ReleaseBlockedSets(); + await store.FlushAsync(); + + SubscriptionRetransmissionState? state = await store.LoadRetransmissionStateAsync(subscriptionId); + + Assert.That(state, Is.Null); + Assert.That(await CountKeysAsync(inner, RetransmissionPrefixFor(subscriptionId)), Is.Zero); + } + + [Test] + public async Task ReusedSubscriptionIdDoesNotLoadDeletedRetransmissionStateAsync() + { + const uint subscriptionId = 707; + using var kv = new InMemorySharedKeyValueStore(); + await using var store = CreateStore(kv); + await using var backup = CreateStore(kv); + + await store.StoreSubscriptionsAsync([NewSubscription(subscriptionId, 77)]); + store.StoreRetransmissionState(subscriptionId, 4, [NewNotification(1), NewNotification(2), NewNotification(3)]); + await store.FlushAsync(); + await store.StoreSubscriptionsAsync([]); + await store.FlushAsync(); + await store.StoreSubscriptionsAsync([NewSubscription(subscriptionId, 78)]); + + SubscriptionRetransmissionState? deletedState = await backup.LoadRetransmissionStateAsync(subscriptionId); + store.StoreRetransmissionState(subscriptionId, 10, [NewNotification(9)]); + await store.FlushAsync(); + SubscriptionRetransmissionState? newState = await backup.LoadRetransmissionStateAsync(subscriptionId); + + Assert.That(deletedState, Is.Null); + Assert.That(newState, Is.Not.Null); + Assert.That(newState!.NextSequenceNumber, Is.EqualTo(10)); + Assert.That( + newState.SentMessages.Memory.ToArray().Select(m => m.SequenceNumber), + Is.EqualTo(new uint[] { 9 })); + } + + [Test] + public void ConstructorValidatesArguments() + { + IServiceMessageContext context = CreateContext(); + var kvMock = new Mock(); + + Assert.That( + () => new SharedKeyValueSubscriptionStore(null!, context), + Throws.ArgumentNullException); + Assert.That( + () => new SharedKeyValueSubscriptionStore(kvMock.Object, null!), + Throws.ArgumentNullException); + } + + [Test] + public void RestoreQueueFallbacksReturnNull() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + + Assert.That(store.RestoreDataChangeMonitoredItemQueue(1), Is.Null); + Assert.That(store.RestoreEventMonitoredItemQueue(1), Is.Null); + } + + [Test] + public void OnSubscriptionRestoreCompleteValidatesArguments() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + + Assert.That( + async () => await store.OnSubscriptionRestoreCompleteAsync(null!), + Throws.ArgumentNullException); + } + + [Test] + public void DefinitionCodecRoundTripsAllSupportedFilterShapes() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + StoredSubscription subscription = NewSubscription(900, 90); + subscription.MonitoredItems = + [ + NewItemWithFilter(900, 90, null), + NewItemWithFilter(900, 91, new EventFilter()), + NewItemWithFilter(900, 92, new AggregateFilter + { + AggregateType = ObjectIds.AggregateFunction_Average, + ProcessingInterval = 1000, + StartTime = new DateTimeUtc(638000000000000000) + }) + ]; + + MethodInfo encode = GetPrivateMethod("Encode", typeof(StoredSubscription)); + MethodInfo decode = GetPrivateMethod("Decode", typeof(ByteString)); + var payload = (ByteString)encode.Invoke(store, new object[] { subscription })!; + var decoded = (StoredSubscription)decode.Invoke(store, new object[] { payload })!; + List decodedItems = decoded.MonitoredItems.ToList(); + + Assert.That(decodedItems, Has.Count.EqualTo(3)); + Assert.That(decodedItems[0].FilterToUse, Is.Null); + Assert.That(decodedItems[1].FilterToUse, Is.TypeOf()); + Assert.That(decodedItems[2].FilterToUse, Is.TypeOf()); + } + + [Test] + public void DefinitionDecodeRejectsUnsupportedVersion() + { + using var kv = new InMemorySharedKeyValueStore(); + SharedKeyValueSubscriptionStore store = CreateStore(kv); + using var encoder = new BinaryEncoder(CreateContext()); + encoder.WriteInt32(null, 999); + ByteString payload = ByteString.From(encoder.CloseAndReturnBuffer()!); + MethodInfo decode = GetPrivateMethod("Decode", typeof(ByteString)); + + TargetInvocationException? ex = Assert.Throws( + () => decode.Invoke(store, new object[] { payload })); + + Assert.That(ex!.InnerException, Is.TypeOf()); + } + + [Test] + public async Task ContinuationPointLoadIgnoresUnsupportedEnvelopeVersionAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var store = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + Guid continuationPointId = Guid.NewGuid(); + using var encoder = new BinaryEncoder(CreateContext()); + encoder.WriteInt32(null, 999); + await kv.SetAsync( + SharedKeyValueSubscriptionStore.ContinuationPointKeyFor( + sessionId, + ContinuationPointKind.Browse, + continuationPointId), + ByteString.From(encoder.CloseAndReturnBuffer()!)); + + ArrayOf envelopes = + await store.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Is.Empty); + } + + [Test] + public async Task ContinuationPointLoadIgnoresInvalidEnvelopeIdAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + await using var store = CreateStore(kv); + NodeId sessionId = new NodeId(Guid.NewGuid(), 1); + Guid continuationPointId = Guid.NewGuid(); + using var encoder = new BinaryEncoder(CreateContext()); + encoder.WriteInt32(null, 1); + encoder.WriteByteString(null, ByteString.From(new byte[] { 1, 2, 3 })); + await kv.SetAsync( + SharedKeyValueSubscriptionStore.ContinuationPointKeyFor( + sessionId, + ContinuationPointKind.Browse, + continuationPointId), + ByteString.From(encoder.CloseAndReturnBuffer()!)); + + ArrayOf envelopes = + await store.LoadContinuationPointsAsync(sessionId); + + Assert.That(envelopes, Is.Empty); + } + + private static SharedKeyValueSubscriptionStore CreateStore(InMemorySharedKeyValueStore kv) + { + return new SharedKeyValueSubscriptionStore(kv, CreateContext()); + } + + private static ServiceMessageContext CreateContext() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext context = ServiceMessageContext.CreateEmpty(telemetry); + context.NamespaceUris.GetIndexOrAppend("urn:test:subscriptions"); + return context; + } + + private static StoredSubscription NewSubscription(uint subscriptionId, uint monitoredItemId) + { + return new StoredSubscription + { + Id = subscriptionId, + IsDurable = true, + LifetimeCounter = 2, + MaxLifetimeCount = 120, + MaxKeepaliveCount = 12, + MaxMessageCount = 8, + MaxNotificationsPerPublish = 99, + PublishingInterval = 250, + Priority = 7, + LastSentMessage = 0, + SequenceNumber = 0, + SentMessages = [], + UserIdentityToken = new AnonymousIdentityToken(), + MonitoredItems = [NewItem(subscriptionId, monitoredItemId)] + }; + } + + private static StoredMonitoredItem NewItem(uint subscriptionId, uint monitoredItemId) + { + var filter = new DataChangeFilter + { + Trigger = DataChangeTrigger.StatusValue, + DeadbandType = (uint)DeadbandType.Absolute, + DeadbandValue = 1.5 + }; + + return new StoredMonitoredItem + { + IsRestored = false, + AlwaysReportUpdates = true, + AttributeId = Attributes.Value, + ClientHandle = 987, + DiagnosticsMasks = DiagnosticsMasks.OperationAll, + DiscardOldest = true, + Encoding = new QualifiedName(BrowseNames.DefaultXml, 0), + Id = monitoredItemId, + IndexRange = "1:3", + ParsedIndexRange = NumericRange.Parse("1:3"), + IsDurable = true, + LastError = ServiceResult.Good, + LastValue = new DataValue(new Variant(42)), + MonitoringMode = MonitoringMode.Reporting, + NodeId = new NodeId("Temperature", NamespaceIndex), + FilterToUse = filter, + OriginalFilter = filter, + QueueSize = 3, + Range = 100, + SamplingInterval = 50, + SourceSamplingInterval = 25, + SubscriptionId = subscriptionId, + TimestampsToReturn = TimestampsToReturn.Both, + TypeMask = 1 + }; + } + + private static StoredMonitoredItem NewItemWithFilter( + uint subscriptionId, + uint monitoredItemId, + MonitoringFilter? filter) + { + StoredMonitoredItem item = NewItem(subscriptionId, monitoredItemId); + item.FilterToUse = filter!; + item.OriginalFilter = filter!; + return item; + } + + private static MethodInfo GetPrivateMethod(string name, params Type[] parameterTypes) + { + MethodInfo? method = typeof(SharedKeyValueSubscriptionStore).GetMethod( + name, + BindingFlags.Instance | BindingFlags.NonPublic, + null, + parameterTypes, + null); + Assert.That(method, Is.Not.Null); + return method!; + } + + private static NotificationMessage NewNotification(uint sequenceNumber) + { + var notification = new DataChangeNotification + { + MonitoredItems = + [ + new MonitoredItemNotification + { + ClientHandle = sequenceNumber, + Value = new DataValue(new Variant((int)sequenceNumber)) + } + ] + }; + + return new NotificationMessage + { + SequenceNumber = sequenceNumber, + PublishTime = DateTimeUtc.Now, + NotificationData = [new ExtensionObject(notification)] + }; + } + + private static string RetransmissionPrefixFor(uint subscriptionId) + { + string messageKey = SharedKeyValueSubscriptionStore.RetransmissionMessageKeyFor(subscriptionId, 0); + return messageKey.Substring(0, messageKey.Length - 10); + } + + private static async Task CountKeysAsync(InMemorySharedKeyValueStore store, string keyPrefix) + { + int count = 0; + await foreach (KeyValuePair pair in store.ScanAsync(keyPrefix)) + { + _ = pair; + count++; + } + (bool found, _) = await store.TryGetAsync( + SharedKeyValueSubscriptionStore.RetransmissionStateKeyFor( + uint.Parse(keyPrefix.Split('/')[1], System.Globalization.CultureInfo.InvariantCulture))); + return found ? count + 1 : count; + } + + private static void AssertSubscription(StoredSubscription actual, StoredSubscription expected) + { + Assert.That(actual.Id, Is.EqualTo(expected.Id)); + Assert.That(actual.IsDurable, Is.EqualTo(expected.IsDurable)); + Assert.That(actual.MaxLifetimeCount, Is.EqualTo(expected.MaxLifetimeCount)); + Assert.That(actual.MaxKeepaliveCount, Is.EqualTo(expected.MaxKeepaliveCount)); + Assert.That(actual.MaxNotificationsPerPublish, Is.EqualTo(expected.MaxNotificationsPerPublish)); + Assert.That(actual.PublishingInterval, Is.EqualTo(expected.PublishingInterval)); + Assert.That(actual.Priority, Is.EqualTo(expected.Priority)); + } + + private static void AssertMonitoredItem(StoredMonitoredItem actual, StoredMonitoredItem expected) + { + Assert.That(actual.SubscriptionId, Is.EqualTo(expected.SubscriptionId)); + Assert.That(actual.Id, Is.EqualTo(expected.Id)); + Assert.That(actual.NodeId, Is.EqualTo(expected.NodeId)); + Assert.That(actual.AttributeId, Is.EqualTo(expected.AttributeId)); + Assert.That(actual.MonitoringMode, Is.EqualTo(expected.MonitoringMode)); + Assert.That(actual.SamplingInterval, Is.EqualTo(expected.SamplingInterval)); + Assert.That(actual.QueueSize, Is.EqualTo(expected.QueueSize)); + Assert.That(actual.DiscardOldest, Is.EqualTo(expected.DiscardOldest)); + Assert.That(actual.FilterToUse, Is.TypeOf()); + Assert.That(((DataChangeFilter)actual.FilterToUse).DeadbandValue, Is.EqualTo(1.5)); + } + + private static bool Contains(byte[] haystack, byte[] needle) + { + for (int ii = 0; ii + needle.Length <= haystack.Length; ii++) + { + bool match = true; + for (int jj = 0; jj < needle.Length; jj++) + { + if (haystack[ii + jj] != needle[jj]) + { + match = false; + break; + } + } + if (match) + { + return true; + } + } + return false; + } + + private static byte[] MakeKey(byte seed) + { + byte[] key = new byte[32]; + for (int ii = 0; ii < key.Length; ii++) + { + key[ii] = (byte)(seed + ii); + } + return key; + } + + private sealed class AsyncSharedKeyValueStore : ISharedKeyValueStore + { + public AsyncSharedKeyValueStore(ISharedKeyValueStore inner) + { + m_inner = inner; + } + + public async ValueTask<(bool Found, ByteString Value)> TryGetAsync( + string key, + CancellationToken ct = default) + { + await Task.Yield(); + return await m_inner.TryGetAsync(key, ct); + } + + public async ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) + { + await Task.Yield(); + await m_inner.SetAsync(key, value, ct); + } + + public async ValueTask CompareAndSwapAsync( + string key, + ByteString expected, + ByteString value, + CancellationToken ct = default) + { + await Task.Yield(); + return await m_inner.CompareAndSwapAsync(key, expected, value, ct); + } + + public async ValueTask DeleteAsync(string key, CancellationToken ct = default) + { + await Task.Yield(); + return await m_inner.DeleteAsync(key, ct); + } + + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + await Task.Yield(); + await foreach (KeyValuePair pair in m_inner.ScanAsync(keyPrefix, ct)) + { + yield return pair; + } + } + + public async IAsyncEnumerable WatchAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + await Task.Yield(); + await foreach (KeyValueChange change in m_inner.WatchAsync(keyPrefix, ct)) + { + yield return change; + } + } + + private readonly ISharedKeyValueStore m_inner; + } + + private sealed class BlockingRetransmissionSetStore : ISharedKeyValueStore + { + public BlockingRetransmissionSetStore(ISharedKeyValueStore inner, uint subscriptionId) + { + m_inner = inner; + m_keyPrefix = "subscription-retransmission/" + + subscriptionId.ToString("D", System.Globalization.CultureInfo.InvariantCulture) + + "/"; + } + + public ValueTask<(bool Found, ByteString Value)> TryGetAsync( + string key, + CancellationToken ct = default) + { + return m_inner.TryGetAsync(key, ct); + } + + public async ValueTask SetAsync(string key, ByteString value, CancellationToken ct = default) + { + if (key.StartsWith(m_keyPrefix, StringComparison.Ordinal) && + Interlocked.Exchange(ref m_blockedSetCount, 1) == 0) + { + m_blocked.SetResult(true); + await m_release.Task.ConfigureAwait(false); + } + + await m_inner.SetAsync(key, value, ct).ConfigureAwait(false); + } + + public ValueTask CompareAndSwapAsync( + string key, + ByteString expected, + ByteString value, + CancellationToken ct = default) + { + return m_inner.CompareAndSwapAsync(key, expected, value, ct); + } + + public ValueTask DeleteAsync(string key, CancellationToken ct = default) + { + return m_inner.DeleteAsync(key, ct); + } + + public async IAsyncEnumerable> ScanAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + await foreach (KeyValuePair pair in m_inner.ScanAsync(keyPrefix, ct)) + { + yield return pair; + } + } + + public async IAsyncEnumerable WatchAsync( + string keyPrefix, + [EnumeratorCancellation] CancellationToken ct = default) + { + await foreach (KeyValueChange change in m_inner.WatchAsync(keyPrefix, ct)) + { + yield return change; + } + } + + public async Task WaitForBlockedSetAsync() + { + Task completed = await Task.WhenAny(m_blocked.Task, Task.Delay(TimeSpan.FromSeconds(10))) + .ConfigureAwait(false); + Assert.That(completed, Is.SameAs(m_blocked.Task)); + } + + public void ReleaseBlockedSets() + { + m_release.SetResult(true); + } + + private readonly ISharedKeyValueStore m_inner; + private readonly string m_keyPrefix; + private readonly TaskCompletionSource m_blocked = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource m_release = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int m_blockedSetCount; + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Values/DistributedValueCacheTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Values/DistributedValueCacheTests.cs new file mode 100644 index 0000000000..6428678371 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Values/DistributedValueCacheTests.cs @@ -0,0 +1,110 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class DistributedValueCacheTests + { + private const ushort NamespaceIndex = 1; + private IServiceMessageContext m_messageContext = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:cache"); + m_messageContext = messageContext; + } + + [Test] + public async Task CachedValueIsFreshWithinMaxAgeAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + var cache = new DistributedValueCache(new InMemoryNodeStateStore(kv, m_messageContext), time); + var nodeId = new NodeId("v", NamespaceIndex); + + await cache.CacheAsync(nodeId, new DataValue(new Variant(1.5), StatusCodes.Good, time.GetUtcNow())); + (bool fresh, DataValue value) = await cache.TryGetAsync(nodeId, TimeSpan.FromMinutes(1)); + + Assert.That(fresh, Is.True); + Assert.That(value.WrappedValue, Is.EqualTo(new Variant(1.5))); + } + + [Test] + public async Task CachedValueIsStaleBeyondMaxAgeButStillReturnedAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + var cache = new DistributedValueCache(new InMemoryNodeStateStore(kv, m_messageContext), time); + var nodeId = new NodeId("v", NamespaceIndex); + + await cache.CacheAsync(nodeId, new DataValue(new Variant(1.5), StatusCodes.Good, time.GetUtcNow())); + time.Advance(TimeSpan.FromMinutes(2)); + (bool fresh, DataValue value) = await cache.TryGetAsync(nodeId, TimeSpan.FromMinutes(1)); + + Assert.That(fresh, Is.False); + Assert.That(value.WrappedValue, Is.EqualTo(new Variant(1.5))); + } + + [Test] + public async Task MissingValueReportsNotFreshAndNullAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var cache = new DistributedValueCache(new InMemoryNodeStateStore(kv, m_messageContext)); + + (bool fresh, DataValue value) = await cache.TryGetAsync( + new NodeId("missing", NamespaceIndex), TimeSpan.FromMinutes(1)); + + Assert.That(fresh, Is.False); + Assert.That(value.IsNull, Is.True); + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Redundancy.Server.Tests/Values/DistributedValueParticipationTests.cs b/Tests/Opc.Ua.Redundancy.Server.Tests/Values/DistributedValueParticipationTests.cs new file mode 100644 index 0000000000..8021bc90f2 --- /dev/null +++ b/Tests/Opc.Ua.Redundancy.Server.Tests/Values/DistributedValueParticipationTests.cs @@ -0,0 +1,159 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2007: tests run without a SynchronizationContext; ConfigureAwait(false) +// adds noise without a behavioural benefit. Disabled file-level for the suite. +#pragma warning disable CA2007 + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Tests; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Server.Tests.Redundancy +{ + /// + /// Unit tests for . + /// + [TestFixture] + [Category("Distributed")] + [Parallelizable(ParallelScope.All)] + public class DistributedValueParticipationTests + { + private const ushort NamespaceIndex = 1; + private IServiceMessageContext m_messageContext = null!; + private SystemContext m_systemContext = null!; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + ITelemetryContext telemetry = NUnitTelemetryContext.Create(); + ServiceMessageContext messageContext = ServiceMessageContext.CreateEmpty(telemetry); + messageContext.NamespaceUris.GetIndexOrAppend("urn:test:participation"); + m_messageContext = messageContext; + m_systemContext = new SystemContext(telemetry) + { + NamespaceUris = messageContext.NamespaceUris, + ServerUris = messageContext.ServerUris, + EncodeableFactory = messageContext.Factory + }; + } + + [Test] + public async Task ReadThroughServesFreshCacheWithoutLiveReadAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + var cache = new DistributedValueCache(new InMemoryNodeStateStore(kv, m_messageContext), time); + var nodeId = new NodeId("v", NamespaceIndex); + await cache.CacheAsync(nodeId, new DataValue(new Variant(2.0), StatusCodes.Good, time.GetUtcNow())); + + int liveCalls = 0; + DataValue result = await DistributedValueParticipation.ReadThroughAsync( + cache, nodeId, TimeSpan.FromMinutes(1), LiveRead); + + Assert.That(liveCalls, Is.Zero, "fresh cached value must be served without a live read"); + Assert.That(result.WrappedValue, Is.EqualTo(new Variant(2.0))); + + ValueTask LiveRead(CancellationToken ct) + { + liveCalls++; + return new ValueTask(new DataValue(new Variant(99.0), StatusCodes.Good, time.GetUtcNow())); + } + } + + [Test] + public async Task ReadThroughReadsLiveWhenStaleAndCachesAsync() + { + var time = new FakeTimeProvider(); + using var kv = new InMemorySharedKeyValueStore(); + var cache = new DistributedValueCache(new InMemoryNodeStateStore(kv, m_messageContext), time); + var nodeId = new NodeId("v", NamespaceIndex); + + int liveCalls = 0; + DataValue result = await DistributedValueParticipation.ReadThroughAsync( + cache, nodeId, TimeSpan.FromMinutes(1), LiveRead); + + Assert.That(liveCalls, Is.EqualTo(1)); + Assert.That(result.WrappedValue, Is.EqualTo(new Variant(7.0))); + + (bool fresh, DataValue cached) = await cache.TryGetAsync(nodeId, TimeSpan.FromMinutes(1)); + Assert.That(fresh, Is.True, "the live value should now be cached and fresh"); + Assert.That(cached.WrappedValue, Is.EqualTo(new Variant(7.0))); + + ValueTask LiveRead(CancellationToken ct) + { + liveCalls++; + return new ValueTask(new DataValue(new Variant(7.0), StatusCodes.Good, time.GetUtcNow())); + } + } + + [Test] + public async Task EnableParticipationWiresWriteThroughAndCachedReadAsync() + { + using var kv = new InMemorySharedKeyValueStore(); + var cache = new DistributedValueCache(new InMemoryNodeStateStore(kv, m_messageContext)); + var nodeId = new NodeId("wired", NamespaceIndex); + var variable = new BaseDataVariableState(null) + { + NodeId = nodeId, + BrowseName = new QualifiedName("Wired", NamespaceIndex), + DisplayName = new LocalizedText("Wired"), + DataType = DataTypeIds.Double, + ValueRank = ValueRanks.Scalar, + Value = new Variant(0.0) + }; + + int liveCalls = 0; + variable.EnableDistributedValueParticipation(cache, TimeSpan.FromMinutes(1), LiveRead); + + AttributeWriteResult writeResult = await variable.OnWriteValueAsync!( + m_systemContext, variable, default, new Variant(8.0), default); + Assert.That(ServiceResult.IsGood(writeResult.Result), Is.True); + + AttributeReadResult readResult = await variable.OnReadValueAsync!( + m_systemContext, variable, default, new QualifiedName(), default); + + Assert.That(readResult.Value, Is.EqualTo(new Variant(8.0)), "the written value is served from the cache"); + Assert.That(liveCalls, Is.Zero, "a fresh written value must be served without a live read"); + + ValueTask LiveRead(CancellationToken ct) + { + liveCalls++; + return new ValueTask(new DataValue(new Variant(123.0), StatusCodes.Good, DateTimeUtc.Now)); + } + } + } +} \ No newline at end of file diff --git a/Tests/Opc.Ua.Server.Tests/Opc.Ua.Server.Tests.csproj b/Tests/Opc.Ua.Server.Tests/Opc.Ua.Server.Tests.csproj index d361dee44c..c35ea1fe7d 100644 --- a/Tests/Opc.Ua.Server.Tests/Opc.Ua.Server.Tests.csproj +++ b/Tests/Opc.Ua.Server.Tests/Opc.Ua.Server.Tests.csproj @@ -12,6 +12,7 @@ + @@ -43,6 +44,7 @@ + diff --git a/Tests/Opc.Ua.Server.Tests/SubscriptionLifecycleTests.cs b/Tests/Opc.Ua.Server.Tests/SubscriptionLifecycleTests.cs index a6ae82e9ca..42b603b5fd 100644 --- a/Tests/Opc.Ua.Server.Tests/SubscriptionLifecycleTests.cs +++ b/Tests/Opc.Ua.Server.Tests/SubscriptionLifecycleTests.cs @@ -115,10 +115,15 @@ private OperationContext CreateOperationContext() private static void InjectSentMessages(Subscription subscription, params NotificationMessage[] messages) { - FieldInfo field = typeof(Subscription).GetField("m_sentMessages", + FieldInfo queueField = typeof(Subscription).GetField("m_messageQueue", + BindingFlags.NonPublic | BindingFlags.Instance) + ?? throw new InvalidOperationException("Field m_messageQueue not found"); + object messageQueue = queueField.GetValue(subscription) + ?? throw new InvalidOperationException("m_messageQueue is null"); + FieldInfo field = messageQueue.GetType().GetField("m_sentMessages", BindingFlags.NonPublic | BindingFlags.Instance) ?? throw new InvalidOperationException("Field m_sentMessages not found"); - var sentMessages = (List)field.GetValue(subscription); + var sentMessages = (List)field.GetValue(messageQueue); sentMessages.AddRange(messages); } diff --git a/Tests/Opc.Ua.Sessions.Tests/ChannelManagerSessionLifecycleIntegrationTests.cs b/Tests/Opc.Ua.Sessions.Tests/ChannelManagerSessionLifecycleIntegrationTests.cs index 05db3dcc85..5352f9fbbf 100644 --- a/Tests/Opc.Ua.Sessions.Tests/ChannelManagerSessionLifecycleIntegrationTests.cs +++ b/Tests/Opc.Ua.Sessions.Tests/ChannelManagerSessionLifecycleIntegrationTests.cs @@ -362,7 +362,7 @@ public ValueTask FetchRedundancyInfoAsync( return new ValueTask( new ServerRedundancyInfo { - Mode = RedundancyMode.Cold, + Mode = RedundancySupport.Cold, ServiceLevel = 200, RedundantServers = [ @@ -376,6 +376,16 @@ public ValueTask FetchRedundancyInfoAsync( }); } + public ServerFailoverDecision ShouldFailover( + ServerRedundancyInfo redundancyInfo, + ConfiguredEndpoint currentEndpoint) + { + return new ServerFailoverDecision( + isFailoverWarranted: true, + DateTime.MinValue, + "Test handler warrants failover to the configured standby."); + } + public ConfiguredEndpoint SelectFailoverTarget( ServerRedundancyInfo redundancyInfo, ConfiguredEndpoint currentEndpoint) diff --git a/Tests/Opc.Ua.Sessions.Tests/DistributedSessionFailoverIntegrationTests.cs b/Tests/Opc.Ua.Sessions.Tests/DistributedSessionFailoverIntegrationTests.cs new file mode 100644 index 0000000000..317730f8a7 --- /dev/null +++ b/Tests/Opc.Ua.Sessions.Tests/DistributedSessionFailoverIntegrationTests.cs @@ -0,0 +1,285 @@ +/* ======================================================================== + * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. + * + * OPC Foundation MIT License 1.00 + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * The complete license agreement can be found here: + * http://opcfoundation.org/License/MIT/1.00/ + * ======================================================================*/ + +// CA2000: integration-test disposables are released by helper cleanup paths. +// CA2007: tests run without a SynchronizationContext. +// CA2016: cleanup intentionally ignores the test cancellation token. +#pragma warning disable CA2000, CA2007, CA2016 + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Opc.Ua.Client; +using Opc.Ua.Client.TestFramework; +using Opc.Ua.Server; +using Opc.Ua.Redundancy.Server; +using Opc.Ua.Server.TestFramework; +using Quickstarts.ReferenceServer; +using ManagedSessionType = Opc.Ua.Client.ManagedSession; +using Opc.Ua.Redundancy; + +namespace Opc.Ua.Sessions.Tests +{ + /// + /// End-to-end secured failover test for the distributed-HA feature: two + /// servers share one session store via , + /// and a client with token-reuse failover fails over from the + /// active to the standby. The standby restores the mirrored session and + /// re-activates it with the reused authentication token, so the client's + /// SessionId is preserved (a fresh re-authentication would change it). + /// + [TestFixture] + [Category("Client")] + [Category("ManagedSession")] + [Category("Distributed")] + [SetCulture("en-us")] + [SetUICulture("en-us")] + [NonParallelizable] + public sealed class DistributedSessionFailoverIntegrationTests : ClientTestFramework + { + [OneTimeSetUp] + public override Task OneTimeSetUpAsync() + { + SupportsExternalServerUrl = true; + SingleSession = false; + // Secured client fixture; the per-test HA servers are created below. + return OneTimeSetUpCoreAsync(securityNone: false); + } + + [OneTimeTearDown] + public override Task OneTimeTearDownAsync() + { + return base.OneTimeTearDownAsync(); + } + + [SetUp] + public override Task SetUpAsync() + { + return base.SetUpAsync(); + } + + [TearDown] + public override Task TearDownAsync() + { + return base.TearDownAsync(); + } + + [Test] + [Order(100)] + [CancelAfter(180_000)] + public async Task TokenReuseFailoverRestoresSessionOnStandbyAsync(CancellationToken ct) + { + using var sharedStore = new InMemorySharedKeyValueStore(); + using var protector = new AesCbcHmacRecordProtector(MakeKey()); + var factory = new DistributedSessionManagerFactory( + sharedStore, + protector, + new DistributedSessionOptions { EnableFastReconnect = true }); + + ServerFixture? fixtureA = null; + ServerFixture? fixtureB = null; + ManagedSessionType? session = null; + + try + { + (fixtureA, Uri urlA) = await StartHaServerAsync(factory).ConfigureAwait(false); + (fixtureB, Uri urlB) = await StartHaServerAsync(factory).ConfigureAwait(false); + + ConfiguredEndpoint endpointA = await ClientFixture + .GetEndpointAsync(urlA, SecurityPolicies.Basic256Sha256) + .ConfigureAwait(false); + ConfiguredEndpoint endpointB = await ClientFixture + .GetEndpointAsync(urlB, SecurityPolicies.Basic256Sha256) + .ConfigureAwait(false); + + var redundancyHandler = new FailoverRedundancyHandler(endpointB); + session = await new ManagedSessionBuilder(ClientFixture.Config, Telemetry) + .UseEndpoint(endpointA) + .WithSessionName(nameof(TokenReuseFailoverRestoresSessionOnStandbyAsync)) + .WithSessionTimeout(TimeSpan.FromSeconds(60)) + .WithServerRedundancy(redundancyHandler) + .WithTokenReuseFailover() + .WithReconnectPolicy(p => p with + { + Strategy = BackoffStrategy.Constant, + InitialDelay = TimeSpan.FromMilliseconds(50), + MaxRetries = 1, + JitterFactor = 0.0 + }) + .ConnectAsync(ct) + .ConfigureAwait(false); + + // Session is live on the active server. + DataValue stateBefore = await session + .ReadValueAsync(VariableIds.Server_ServerStatus_State, ct) + .ConfigureAwait(false); + Assert.That(StatusCode.IsGood(stateBefore.StatusCode), Is.True); + + NodeId sessionIdBefore = session.InnerSession.SessionId; + Assert.That(sessionIdBefore.IsNull, Is.False); + + // Force a failover: make the in-place reconnect fail so the state + // machine selects the redundant endpoint B. + var reconnected = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + session.ConnectionStateChanged += (_, e) => + { + if (e.PreviousState == ConnectionState.Failover && + e.NewState == ConnectionState.Connected) + { + reconnected.TrySetResult(true); + } + }; + + session.StateMachine.ReconnectWithBudgetAsync = (_, _) => + Task.FromResult(new ServiceResult(StatusCodes.BadNotConnected)); + session.StateMachine.TriggerReconnect(); + + Assert.That( + await reconnected.Task.WaitAsync(TimeSpan.FromSeconds(60), ct).ConfigureAwait(false), + Is.True, + "the client should fail over to the standby server."); + + // The session works on the standby... + DataValue stateAfter = await session + .ReadValueAsync(VariableIds.Server_ServerStatus_State, ct) + .ConfigureAwait(false); + Assert.That(StatusCode.IsGood(stateAfter.StatusCode), Is.True); + + // ...and its SessionId is preserved, proving the standby restored + // the mirrored session and re-activated it with the reused token + // rather than creating a brand-new session (re-auth fallback). + Assert.That( + session.InnerSession.SessionId, + Is.EqualTo(sessionIdBefore), + "token-reuse failover must preserve the SessionId (no fresh CreateSession)."); + Assert.That(redundancyHandler.SelectCount, Is.GreaterThanOrEqualTo(1)); + } + finally + { + if (session != null) + { + await session.CloseAsync(CancellationToken.None).ConfigureAwait(false); + await session.DisposeAsync().ConfigureAwait(false); + } + if (fixtureA != null) + { + await fixtureA.StopAsync().ConfigureAwait(false); + } + if (fixtureB != null) + { + await fixtureB.StopAsync().ConfigureAwait(false); + } + } + } + + private async Task<(ServerFixture Fixture, Uri Url)> StartHaServerAsync( + DistributedSessionManagerFactory factory) + { + var fixture = new ServerFixture(telemetry => + { + var server = new ReferenceServer(telemetry); + server.SessionManagerFactory = factory; + return server; + }) + { + UriScheme = Utils.UriSchemeOpcTcp, + SecurityNone = false, + AutoAccept = true, + OperationLimits = true + }; + + await fixture.StartAsync().ConfigureAwait(false); + return (fixture, new Uri($"{Utils.UriSchemeOpcTcp}://localhost:{fixture.Port}")); + } + + private static byte[] MakeKey() + { + byte[] key = new byte[32]; + for (int i = 0; i < key.Length; i++) + { + key[i] = (byte)(i + 7); + } + return key; + } + + private sealed class FailoverRedundancyHandler : IServerRedundancyHandler + { + public FailoverRedundancyHandler(ConfiguredEndpoint target) + { + m_target = target; + } + + public int SelectCount { get; private set; } + + public ValueTask FetchRedundancyInfoAsync( + Client.ISession session, + CancellationToken ct = default) + { + return new ValueTask(new ServerRedundancyInfo + { + Mode = RedundancySupport.Hot, + ServiceLevel = 200, + RedundantServers = + [ + new RedundantServer + { + ServerUri = m_target.EndpointUrl?.ToString() ?? "urn:standby", + ServerState = ServerState.Running, + ServiceLevel = 250 + } + ] + }); + } + + public ServerFailoverDecision ShouldFailover( + ServerRedundancyInfo redundancyInfo, + ConfiguredEndpoint currentEndpoint) + { + return new ServerFailoverDecision( + isFailoverWarranted: true, + DateTime.MinValue, + "Test handler warrants failover to the configured standby."); + } + + public ConfiguredEndpoint SelectFailoverTarget( + ServerRedundancyInfo redundancyInfo, + ConfiguredEndpoint currentEndpoint) + { + SelectCount++; + return m_target; + } + + private readonly ConfiguredEndpoint m_target; + } + } +} diff --git a/Tests/Opc.Ua.Sessions.Tests/ManagedSessionReconnectIntegrationTests.cs b/Tests/Opc.Ua.Sessions.Tests/ManagedSessionReconnectIntegrationTests.cs index 5fdc061cac..a0815ce12a 100644 --- a/Tests/Opc.Ua.Sessions.Tests/ManagedSessionReconnectIntegrationTests.cs +++ b/Tests/Opc.Ua.Sessions.Tests/ManagedSessionReconnectIntegrationTests.cs @@ -1343,7 +1343,7 @@ public ValueTask FetchRedundancyInfoAsync( return new ValueTask( new ServerRedundancyInfo { - Mode = RedundancyMode.Cold, + Mode = RedundancySupport.Cold, ServiceLevel = 200, RedundantServers = [ @@ -1357,6 +1357,18 @@ public ValueTask FetchRedundancyInfoAsync( }); } + public ServerFailoverDecision ShouldFailover( + ServerRedundancyInfo redundancyInfo, + ConfiguredEndpoint currentEndpoint) + { + return new ServerFailoverDecision( + isFailoverWarranted: m_target != null, + DateTime.MinValue, + m_target != null + ? "Test handler warrants failover to the configured standby." + : "No failover target configured."); + } + public ConfiguredEndpoint? SelectFailoverTarget( ServerRedundancyInfo redundancyInfo, ConfiguredEndpoint currentEndpoint) diff --git a/Tests/Opc.Ua.Sessions.Tests/Opc.Ua.Sessions.Tests.csproj b/Tests/Opc.Ua.Sessions.Tests/Opc.Ua.Sessions.Tests.csproj index 13a4313fc9..4bea8c68fe 100644 --- a/Tests/Opc.Ua.Sessions.Tests/Opc.Ua.Sessions.Tests.csproj +++ b/Tests/Opc.Ua.Sessions.Tests/Opc.Ua.Sessions.Tests.csproj @@ -40,6 +40,7 @@ + diff --git a/UA.slnx b/UA.slnx index 9f6e0359e0..a6bf1d763b 100644 --- a/UA.slnx +++ b/UA.slnx @@ -4,11 +4,13 @@ + + @@ -26,6 +28,7 @@ + @@ -66,6 +69,7 @@ + @@ -74,6 +78,9 @@ + + + @@ -199,6 +206,7 @@ + @@ -229,6 +237,8 @@ + + diff --git a/codecov.yml b/codecov.yml index e5cb1f0c68..8a73155cca 100644 --- a/codecov.yml +++ b/codecov.yml @@ -37,6 +37,15 @@ ignore: - "**/obj/**" - "**/bin/**" - "**/*.g.cs" + # Integration-only Kubernetes IO: real HTTP to the API server and the readiness + # HTTP listener / cluster-bound startup tasks are exercised by integration, not + # unit tests (mirrors the Applications/** exemption). The Kubernetes logic + # (models, factory, lease election, peer-slice parsing, readiness mapping, + # builder wiring) stays measured. + - "Libraries/Opc.Ua.Server.Redundancy.K8s/Client/KubernetesHttpApiClient.cs" + - "Libraries/Opc.Ua.Server.Redundancy.K8s/Health/KubernetesReadinessServer.cs" + - "Libraries/Opc.Ua.Server.Redundancy.K8s/Health/KubernetesReadinessStartupTask.cs" + - "Libraries/Opc.Ua.Server.Redundancy.K8s/PeerDiscovery/KubernetesPeerDiscoveryStartupTask.cs" comment: layout: "reach, diff, flags, files" diff --git a/plans/28-distributed-ha-remaining.md b/plans/28-distributed-ha-remaining.md new file mode 100644 index 0000000000..f8403dcf1b --- /dev/null +++ b/plans/28-distributed-ha-remaining.md @@ -0,0 +1,41 @@ +# Distributed / High-Availability Server — Remaining Work + +## Status + +The distributed / high-availability feature is implemented, tested, documented, and shipping on the `nodestatestorage` branch (draft PR OPCFoundation/UA-.NETStandard#3918). This file consolidates the former plans 28–32 down to the work that is **not yet delivered**. Everything else those plans described is done and is described in the shipped docs. + +## Baseline (already delivered — do not re-plan) + +See `Docs/HighAvailability.md` and `Docs/Kubernetes.md` for the full, current design. In summary, the following are complete: + +- Provider-based shared address-space state (topology + values) behind `ISharedKeyValueStore` / `INodeStateStore`, with the address-space synchronizer, opt-in node-manager adapter, and DI/fluent wiring (`UseReplicatedAddressSpace` / `UseDistributedAddressSpace` and the `IServerStartupTask` seam). Default path (no store) is unchanged and zero-overhead. +- Leader election (static, shared-store lease CAS, and native Raft leadership), dynamic `Server.ServiceLevel` (`IServiceLevelProvider`), and `Server.ServerRedundancy` population for both transparent (`CurrentServerId`) and non-transparent (`ServerUriArray` / `RedundantServerArray`) modes. +- Distributed value cache (read/write callback participation with freshness). +- Secure session mirroring: `DistributedSessionManager` via the `ISessionManagerFactory` seam, encrypted + integrity-protected records (`IRecordProtector` / `AesCbcHmacRecordProtector` / `KeyRingRecordProtector`), cross-replica single-use nonce CAS, full `ActivateSession` signature verification on restore (the token is a lookup key only), and restore audit. Safe default is re-auth on failover; mirrored fast reconnect is opt-in. +- Client token-reuse failover (`ManagedSession` / `WithTokenReuseFailover`), network redundancy endpoint alternates, and HotAndMirrored state mirroring with deterministic EventIds (`DeterministicEventIdProvider`). +- Both consistency backends in-package over the NanoMsg transport, selectable via `UseRedundancyConsistency`: **CRDT** (eventual, active/active, leaderless) and **Raft** (`RaftCs`, linearizable strong consistency for `nonce/` / `lease/` / `election/`). Kubernetes wiring via `UseKubernetesRaftConsensus`. +- Libraries `Opc.Ua.Redundancy`, `Opc.Ua.Redundancy.Server`, `Opc.Ua.Redundancy.Client`, `Opc.Ua.Redundancy.K8s`; samples `Applications/RedundantServer` (+ `docker-compose` active/active, active/passive, Raft) and `Applications/RedundantClient`; the `Docs/Kubernetes.md` deployment guide. +- Security findings F1–F7 and F9 from the security assessment are closed in code. + +## Remaining work + +### 1. Lazy topology materialization for very large address spaces — deferred + +Startup and failover-promotion hydration of the shared-store (active/passive) path now streams variable values in bulk (`INodeStateStore.EnumerateValuesAsync`, one pass instead of a read per variable), but node **topology** is still materialized eagerly by `AddressSpaceSynchronizer.SeedOrHydrateAsync` (deserialize + insert every stored node). Snapshot/lazy materialization of the topology (fault-in on browse/read) for very large graphs remains a benchmarked future optimization. The eventual-consistency CRDT path already exchanges a compact state snapshot plus deltas, so this only affects the shared-store path. + +## Delivered in this iteration + +- **Async `ISubscriptionStore` definition-persistence contract.** `StoreSubscriptions`/`RestoreSubscriptions`/`OnSubscriptionRestoreComplete` are now `StoreSubscriptionsAsync`/`RestoreSubscriptionsAsync`/`OnSubscriptionRestoreCompleteAsync` (`ValueTask`-returning, `CancellationToken`). Subscription definitions can now be persisted to an async network backend without a sync-over-async wrapper; `SharedKeyValueSubscriptionStore` awaits its shared-store writes directly instead of fire-and-forget. The per-monitored-item queue-restore hooks stay synchronous (synchronous monitored-item creation path). Documented in `Docs/migrate/2.0.x/sessions-subscriptions.md` and `Docs/HighAvailability.md`. +- **Transparent-redundancy worked sample.** `Applications/RedundantServer/docker-compose.transparent.yml` runs two `REDUNDANCY_MODE=transparent` replicas that share one `ApplicationUri` and one `ApplicationInstanceCertificate` (seeded into a shared PKI volume) and mirror session + address-space state by CRDT gossip, behind an `nginx` TCP load balancer (`nginx.transparent.conf`) that publishes a single virtual endpoint. `Program.cs` gained `HA_APPLICATION_URI` / `HA_SUBJECT_NAME` / `HA_PKI_ROOT` for the shared identity; each replica advertises the load-balancer host (bind-to-any for DNS hosts), so discovery and `CreateSession` echo the single endpoint and a mirrored session resumes on the survivor after a replica fails. Documented in `Applications/RedundantServer/README.md` and `Docs/HighAvailability.md`. +- **Bulk value hydration.** `INodeStateStore.EnumerateValuesAsync` streams the whole value keyspace in one pass; `AddressSpaceSynchronizer.SeedOrHydrateAsync` uses it instead of a `TryReadValueAsync` per variable, so hydrating a large address space no longer costs one round trip per variable against a networked (CRDT/Raft) backend. A first step toward the snapshot-based hydration above. + +## Notes + +- CRDT active/active and Raft strong consistency were listed as "deferred" in the original plans; both are now **delivered** in-package and are therefore intentionally absent from the remaining work above. +- The original "reconnect using just the AuthenticationToken" idea shipped only in its **safe** form (token = lookup key; admission always requires a full `ActivateSession` client-signature check against a single-use mirrored nonce). No token-only reconnect remains to be done. + +## References + +- Shipped docs: `Docs/HighAvailability.md`, `Docs/Kubernetes.md`. +- Code: `Libraries/Opc.Ua.Redundancy`, `Libraries/Opc.Ua.Redundancy.Server`, `Libraries/Opc.Ua.Redundancy.Client`, `Libraries/Opc.Ua.Redundancy.K8s`; `Applications/RedundantServer`, `Applications/RedundantClient`. +- Superseded plans consolidated here (see git history): `plans/28-distributed-ha-node-state.md`, `29-distributed-ha-followup.md`, `30-distributed-ha-session-security.md`, `31-distributed-ha-session-manager.md`, `32-distributed-ha-followup-2.md`.