NCBC-4138: Several Tracing issues

Motivation
==========
Spans were missing modern OTel tags when convention was set to Both or
Modern, cluster name changes weren't propagated, and KV dispatch spans
misattributed the resolved IP as server.address.

Modification
============
- Changed Convention from init-only to re-resolvable; Cluster.cs calls
  ResolveAndSetConvention() after assigning ClusterContext.
- RequestSpanWrapper.ChildSpan now wraps child spans so attributes
  route through SemanticConventionEmitter.
- Added value mapping to SemanticConventionEmitter ("IP.TCP" → "tcp").
- Rewrote UpdateClusterLabelsIfNecessary as MergeClusterLabels: the
  old method copied stale labels onto incoming configs, preventing
  server-provided name/uuid changes from propagating. The new method
  prefers incoming server values and writes the merged result to both
  configs, which is necessary because GlobalConfig is not always
  replaced (the early return in ConfigUpdatedAsync when there are no
  structural changes would otherwise discard the updated labels).
- Threaded logical hostname through MultiplexingConnection/SslConnection
  so AddTags writes logical hostname to net.peer.name and resolved IP
  to new network.peer.address/network.peer.port attributes.

Note: net.peer.name on KV dispatch spans now emits the logical hostname
instead of the resolved IP in all convention modes. This aligns with
the spec but changes existing behavior for consumers parsing this value.

Results
=======
All existing and new unit tests pass. Spans now correctly emit modern
convention tags for both parent and child spans. Observability tests
all pass in FIT.

Change-Id: Ifc65f01543129803e233879f15cda041cc0c53ce
Reviewed-on: https://review.couchbase.org/c/couchbase-net-client/+/242461
Reviewed-by: Jeffry Morris <jeffrymorris@gmail.com>
Tested-by: Build Bot <build@couchbase.com>

Author: davidkelly

src/Couchbase/Cluster.cs

@@ -112,6 +112,7 @@ internal Cluster(ClusterOptions clusterOptions)
             if (_tracer is RequestTracerWrapper wrapper)
             {
                 wrapper.ClusterContext = _context;
+                wrapper.ResolveAndSetConvention();
             }
             _retryStrategy = _context.ServiceProvider.GetRequiredService<IRetryStrategy>();
 

src/Couchbase/Core/ClusterContext.cs

@@ -932,8 +932,12 @@ public async Task ConfigUpdatedAsync(BucketConfig globalConfig)
             }
 #endif
 
-            // Change ClusterLabels if new ones have been received (can come from either Global or Bucket configs)
-            GlobalConfig?.UpdateClusterLabelsIfNecessary(globalConfig);
+            // Merge ClusterLabels: prefer incoming values (latest from server), carry forward current values as fallback.
+            // Both configs are updated so labels survive even if GlobalConfig is replaced.
+            if (GlobalConfig is not null)
+            {
+                globalConfig.MergeClusterLabels(GlobalConfig);
+            }
 
             if (globalConfig.Name == Name)
             {

src/Couchbase/Core/Configuration/Server/BucketConfigExtensions.cs

@@ -36,18 +36,19 @@ public static bool HasClusterNodesChanged(this BucketConfig config, BucketConfig
             return !(config.NodesExt.AreEqual(other.NodesExt) && config.Nodes.AreEqual(other.Nodes));
         }
 
-        public static void UpdateClusterLabelsIfNecessary(this BucketConfig newConfig, BucketConfig oldConfig)
+        public static void MergeClusterLabels(this BucketConfig incomingConfig, BucketConfig currentConfig)
         {
-            if (newConfig.ClusterLabels.Equals(oldConfig.ClusterLabels!)) return;
-            if (newConfig.ClusterLabels.ClusterName is not null)
-            {
-                oldConfig.ClusterLabels.ClusterName = newConfig.ClusterLabels.ClusterName;
-            }
+            if (incomingConfig.ClusterLabels.Equals(currentConfig.ClusterLabels)) return;
 
-            if (newConfig.ClusterLabels.ClusterUuid is not null)
-            {
-                oldConfig.ClusterLabels.ClusterUuid = newConfig.ClusterLabels.ClusterUuid;
-            }
+            // Prefer incoming values (latest from server), fall back to current values
+            var name = incomingConfig.ClusterLabels.ClusterName ?? currentConfig.ClusterLabels.ClusterName;
+            var uuid = incomingConfig.ClusterLabels.ClusterUuid ?? currentConfig.ClusterLabels.ClusterUuid;
+
+            // Apply to both so labels are correct regardless of which config is used going forward
+            incomingConfig.ClusterLabels.ClusterName = name;
+            incomingConfig.ClusterLabels.ClusterUuid = uuid;
+            currentConfig.ClusterLabels.ClusterName = name;
+            currentConfig.ClusterLabels.ClusterUuid = uuid;
         }
         public static bool HasConfigChanges(this BucketConfig newConfig, BucketConfig? oldConfig, string bucketName)
         {

src/Couchbase/Core/Diagnostics/SemanticConventionEmitter.cs

@@ -44,6 +44,37 @@ internal static class SemanticConventionEmitter
         };
 #endif
 
+    // Value mapping: legacy attribute value -> modern attribute value.
+    // Applied only when emitting in Modern or Both mode for string-typed attributes.
+#if NET8_0_OR_GREATER
+    private static readonly FrozenDictionary<string, string> LegacyToModernValues =
+        new Dictionary<string, string>(StringComparer.Ordinal)
+        {
+#else
+    private static readonly Dictionary<string, string> LegacyToModernValues =
+        new Dictionary<string, string>(StringComparer.Ordinal)
+        {
+#endif
+            ["IP.TCP"] = "tcp",
+#if NET8_0_OR_GREATER
+        }.ToFrozenDictionary(StringComparer.Ordinal);
+#else
+        };
+#endif
+
+    /// <summary>
+    /// Maps a legacy string value to its modern equivalent, if one exists.
+    /// </summary>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static T MapValue<T>(T value)
+    {
+        if (value is string strValue && LegacyToModernValues.TryGetValue(strValue, out var modernValue))
+        {
+            return (T)(object)modernValue;
+        }
+        return value;
+    }
+
     /// <summary>
     /// Emits an attribute using the appropriate semantic convention key(s).
     /// The <paramref name="state"/> parameter is forwarded to the callback, allowing
@@ -76,13 +107,13 @@ internal static void EmitAttribute<TState, T>(
         {
             case ObservabilitySemanticConvention.Modern:
                 if (modernKey.Length == 0) return;
-                setAttribute(state, modernKey, value);
+                setAttribute(state, modernKey, MapValue(value));
                 return;
 
             case ObservabilitySemanticConvention.Both:
                 setAttribute(state, key, value);
                 if (modernKey.Length == 0) return;
-                setAttribute(state, modernKey, value);
+                setAttribute(state, modernKey, MapValue(value));
                 return;
 
             default:

src/Couchbase/Core/Diagnostics/Tracing/InnerRequestSpans.cs

@@ -129,6 +129,16 @@ internal static class Attributes
                 /// The operation timeout in milliseconds
                 /// </summary>
                 public const string TimeoutMilliseconds = "timeout_ms";
+
+                /// <summary>
+                ///     The resolved IP address of the remote peer (modern OTel convention).
+                /// </summary>
+                public const string NetworkPeerAddress = "network.peer.address";
+
+                /// <summary>
+                ///     The port of the remote peer (modern OTel convention).
+                /// </summary>
+                public const string NetworkPeerPort = "network.peer.port";
             }
         }
     }

src/Couchbase/Core/Diagnostics/Tracing/RequestSpanWrapper.cs

@@ -56,7 +56,7 @@ public IRequestSpan Parent
     }
     public IRequestSpan ChildSpan(string name)
     {
-        return innerSpan.ChildSpan(name);
+        return new RequestSpanWrapper(innerSpan.ChildSpan(name), clusterLabels, convention);
     }
 
     public bool CanWrite => innerSpan.CanWrite;

src/Couchbase/Core/Diagnostics/Tracing/RequestTracerWrapper.cs

@@ -6,7 +6,15 @@ internal sealed class RequestTracerWrapper(IRequestTracer innerTracer, ClusterCo
     internal ClusterContext ClusterContext = clusterContext;
     internal ClusterLabels ClusterLabels => ClusterContext?.GlobalConfig?.ClusterLabels;
 
-    internal ObservabilitySemanticConvention Convention { get; } = ResolveConvention(clusterContext);
+    internal ObservabilitySemanticConvention Convention { get; private set; } = ResolveConvention(clusterContext);
+
+    /// <summary>
+    /// Re-resolves the Convention from ClusterContext. Call once after ClusterContext is assigned.
+    /// </summary>
+    internal void ResolveAndSetConvention()
+    {
+        Convention = ResolveConvention(ClusterContext);
+    }
 
     private static ObservabilitySemanticConvention ResolveConvention(ClusterContext clusterContext)
     {

src/Couchbase/Core/IO/Connections/ConnectionFactory.cs

@@ -126,10 +126,10 @@ public async Task<IConnection> CreateAndConnectAsync(HostEndpointWithPort hostEn
                 if (!isSecure) throw new AuthenticationException($"The SSL/TLS connection could not be authenticated on [{targetHost}].");
 
                 return new SslConnection(sslStream, _clusterOptions.Tuning.MaximumInFlightOperationsPerConnection,
-                    socket.LocalEndPoint!, socket.RemoteEndPoint!, _sslLogger, _multiplexLogger);
+                    socket.LocalEndPoint!, socket.RemoteEndPoint!, _sslLogger, _multiplexLogger, hostEndpoint.Host);
             }
 
-            return new MultiplexingConnection(socket, _clusterOptions.Tuning.MaximumInFlightOperationsPerConnection, _multiplexLogger);
+            return new MultiplexingConnection(socket, _clusterOptions.Tuning.MaximumInFlightOperationsPerConnection, _multiplexLogger, hostEndpoint.Host);
         }
     }
 }

src/Couchbase/Core/IO/Connections/MultiplexingConnection.cs

@@ -42,20 +42,23 @@ internal sealed class MultiplexingConnection : IConnection
         private readonly string _localHostString;
         private readonly string _remotePortString;
         private readonly string _localPortString;
+        private readonly string _logicalRemoteHostname;
 
         // Connection pooling normally prevents simultaneous writes, but there are cases where they may occur,
         // such as when running Diagnostics pings. We therefore need to prevent them ourselves, as the internal
         // implementation of socket writes may interleave large buffers written from different threads.
         private readonly SemaphoreSlim _writeMutex = new(1);
 
-        public MultiplexingConnection(Socket socket, int maximumInFlightOperations, ILogger<MultiplexingConnection> logger)
+        public MultiplexingConnection(Socket socket, int maximumInFlightOperations,
+            ILogger<MultiplexingConnection> logger, string? logicalRemoteHostname = null)
             : this(new NetworkStream(socket, true), maximumInFlightOperations,
-                socket.LocalEndPoint!, socket.RemoteEndPoint!, logger)
+                socket.LocalEndPoint!, socket.RemoteEndPoint!, logger, logicalRemoteHostname)
         {
         }
 
         public MultiplexingConnection(Stream stream, int maximumInFlightOperations,
-            EndPoint localEndPoint, EndPoint remoteEndPoint, ILogger<MultiplexingConnection> logger)
+            EndPoint localEndPoint, EndPoint remoteEndPoint, ILogger<MultiplexingConnection> logger,
+            string? logicalRemoteHostname = null)
         {
             _stream = stream ?? throw new ArgumentNullException(nameof(stream));
             LocalEndPoint = localEndPoint ?? throw new ArgumentNullException(nameof(localEndPoint));
@@ -71,6 +74,7 @@ public MultiplexingConnection(Stream stream, int maximumInFlightOperations,
             _localHostString = ((IPEndPoint) LocalEndPoint).Address.ToString() ?? DiagnosticsReportProvider.UnknownEndpointValue;
             _remotePortString = ((IPEndPoint) EndPoint).Port.ToStringInvariant();
             _localPortString = ((IPEndPoint) LocalEndPoint).Port.ToStringInvariant();
+            _logicalRemoteHostname = logicalRemoteHostname ?? _remoteHostString;
 
             EndpointState = EndpointState.Connecting;
 
@@ -492,8 +496,10 @@ public void AddTags(IRequestSpan span)
             {
                 span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.LocalHostname, _localHostString);
                 span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.LocalPort, _localPortString);
-                span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.RemoteHostname, _remoteHostString);
+                span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.RemoteHostname, _logicalRemoteHostname);
                 span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.RemotePort, _remotePortString);
+                span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.NetworkPeerAddress, _remoteHostString);
+                span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.NetworkPeerPort, _remotePortString);
                 span.SetAttribute(InnerRequestSpans.DispatchSpan.Attributes.LocalId, ContextId);
             }
         }

src/Couchbase/Core/IO/Connections/SslConnection.cs

@@ -20,13 +20,14 @@ internal sealed class SslConnection : IConnection
         private readonly MultiplexingConnection _multiplexingConnection;
 
         public SslConnection(SslStream stream, int maximumInFlightOperations, EndPoint localEndPoint, EndPoint remoteEndPoint,
-            ILogger<SslConnection> logger, ILogger<MultiplexingConnection> multiplexingLogger)
+            ILogger<SslConnection> logger, ILogger<MultiplexingConnection> multiplexingLogger,
+            string? logicalRemoteHostname = null)
         {
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
             _sslStream = stream ?? throw new ArgumentNullException(nameof(stream));
             _multiplexingConnection = new MultiplexingConnection(_sslStream, maximumInFlightOperations,
-                localEndPoint, remoteEndPoint, multiplexingLogger);
+                localEndPoint, remoteEndPoint, multiplexingLogger, logicalRemoteHostname);
         }
 
         public string ContextId => _multiplexingConnection.ContextId;