remove (or #if) most of the trace info that was added re the write stall;

mgravell · mgravell · commit d137e52b51f0 · 2019-04-02T16:49:17.000+01:00
diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md
@@ -1,5 +1,9 @@
 # Release Notes
 
+## (unreleased)
+
+- performance: resolve intermittent stall in the write-lock that could lead to unexpected timeouts even when at low/reasonable (but concurrent) load
+
 ## 2.0.571
 
 - performance: use new [arena allocation API](https://mgravell.github.io/Pipelines.Sockets.Unofficial/docs/arenas) to avoid `RawResult[]` overhead
diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs
@@ -2152,15 +2152,15 @@ private static async Task<T> ExecuteAsyncImpl_Awaited<T>(ConnectionMultiplexer @
             return tcs == null ? default(T) : await tcs.Task.ForAwait();
         }
 
-        internal Exception GetException(WriteResult result, Message message, ServerEndPoint server, [CallerMemberName] string caller = null)
+        internal Exception GetException(WriteResult result, Message message, ServerEndPoint server)
         {
             switch (result)
             {
                 case WriteResult.Success: return null;
                 case WriteResult.NoConnectionAvailable:
                     return ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, IncludePerformanceCountersInExceptions, message.Command, message, server, GetServerSnapshot());
                 case WriteResult.TimeoutBeforeWrite:
-                    return ExceptionFactory.Timeout(this, "The timeout was reached before the message could be written to the output buffer, and it was not sent", message, server, result, origin: caller);
+                    return ExceptionFactory.Timeout(this, "The timeout was reached before the message could be written to the output buffer, and it was not sent", message, server, result);
                 case WriteResult.WriteFailure:
                 default:
                     return ExceptionFactory.ConnectionFailure(IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, "An unknown error occurred when writing the message", server);
diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs
@@ -181,7 +181,7 @@ internal static string GetLibVersion()
             }
             return _libVersion;
         }
-        internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null, string origin = null)
+        internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null)
         {
             List<Tuple<string, string>> data = new List<Tuple<string, string>> { Tuple.Create("Message", message.CommandAndKey) };
             var sb = new StringBuilder();
@@ -203,16 +203,16 @@ void add(string lk, string sk, string v)
                 }
             }
 
-            if (!string.IsNullOrWhiteSpace(origin)) add("Origin", null, origin);
-
             // Add timeout data, if we have it
             if (result == WriteResult.TimeoutBeforeWrite)
             {
                 add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));
                 try
                 {
+#if DEBUG
                     if (message.QueuePosition >= 0) add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue
                     if ((int)message.ConnectionWriteState >= 0) add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue
+#endif
                     if (message.TryGetPhysicalState(out var state, out var sentDelta, out var receivedDelta))
                     {
                         add("PhysicalState", "phys", state.ToString());
diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs
@@ -1,5 +1,6 @@
 ﻿using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -51,13 +52,18 @@ protected override void WriteImpl(PhysicalConnection physical)
     internal abstract class Message : ICompletable
     {
         public readonly int Db;
+
+#if DEBUG
         internal int QueuePosition { get; private set; }
         internal PhysicalConnection.WriteStatus ConnectionWriteState { get; private set; }
-
+#endif
+        [Conditional("DEBUG")]
         internal void SetBacklogState(int position, PhysicalConnection physical)
         {
+#if DEBUG
             QueuePosition = position;
             ConnectionWriteState = physical?.Status ?? (PhysicalConnection.WriteStatus)(-1);
+#endif
         }
 
         internal const CommandFlags InternalCallFlag = (CommandFlags)128;
@@ -623,8 +629,10 @@ internal bool TrySetResult<T>(T value)
 
         internal void SetEnqueued(PhysicalConnection connection)
         {
+#if DEBUG
             QueuePosition = -1;
             ConnectionWriteState = (PhysicalConnection.WriteStatus)(-1);
+#endif
             SetWriteTime();
             performance?.SetEnqueued();
             _enqueuedTo = connection;
diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs
@@ -643,8 +643,10 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message
                 Multiplexer?.OnInfoMessage($"reentrant call to WriteMessageTakingWriteLock for {message.CommandAndKey}, {existingMessage.CommandAndKey} is still active");
                 return WriteResult.NoConnectionAvailable;
             }
+#if DEBUG
             int startWriteTime = Environment.TickCount;
             try
+#endif
             {
                 physical.SetWriting();
                 var messageIsSent = false;
@@ -679,6 +681,7 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message
                     return WriteMessageToServerInsideWriteLock(physical, message);
                 }
             }
+#if DEBUG
             finally
             {
                 int endWriteTime = Environment.TickCount;
@@ -689,10 +692,12 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message
                     _maxWriteCommand = message?.Command ?? default;
                 }
             }
+#endif
         }
-
+#if DEBUG
         private volatile int _maxWriteTime = -1;
         private RedisCommand _maxWriteCommand;
+#endif
 
         [Obsolete("prefer async")]
         internal WriteResult WriteMessageTakingWriteLockSync(PhysicalConnection physical, Message message)
@@ -755,10 +760,14 @@ private bool PushToBacklog(Message message, bool onlyIfExists)
         private void StartBacklogProcessor()
         {
             var sched = Multiplexer.SocketManager?.SchedulerPool ?? DedicatedThreadPoolPipeScheduler.Default;
+#if DEBUG
             _backlogProcessorRequestedTime = Environment.TickCount;
+#endif
             sched.Schedule(s_ProcessBacklog, _weakRefThis);
         }
+#if DEBUG
         private volatile int _backlogProcessorRequestedTime;
+#endif
 
         private static readonly Action<object> s_ProcessBacklog = s =>
         {
@@ -807,20 +816,26 @@ private void ProcessBacklog()
             LockToken token = default;
             try
             {
+#if DEBUG
                 int tryToAcquireTime = Environment.TickCount;
                 var msToStartWorker = unchecked(tryToAcquireTime - _backlogProcessorRequestedTime);
                 int failureCount = 0;
+#endif
                 while(true)
                 {
                     // try and get the lock; if unsuccessful, check for termination
                     token = _singleWriterMutex.TryWait();
                     if (token) break; // got the lock
                     lock (_backlog) { if (_backlog.Count == 0) return; }
+#if DEBUG
                     failureCount++;
+#endif
                 }
                 _backlogStatus = BacklogStatus.Started;
+#if DEBUG
                 int acquiredTime = Environment.TickCount;
                 var msToGetLock = unchecked(acquiredTime - tryToAcquireTime);
+#endif
 
                 // so now we are the writer; write some things!
                 Message message;
@@ -841,14 +856,15 @@ private void ProcessBacklog()
                         {
                             _backlogStatus = BacklogStatus.RecordingTimeout;
                             var ex = Multiplexer.GetException(WriteResult.TimeoutBeforeWrite, message, ServerEndPoint);
+#if DEBUG // additional tracking
                             ex.Data["Redis-BacklogStartDelay"] = msToStartWorker;
                             ex.Data["Redis-BacklogGetLockDelay"] = msToGetLock;
                             if (failureCount != 0) ex.Data["Redis-BacklogFailCount"] = failureCount;
                             if (_maxWriteTime >= 0) ex.Data["Redis-MaxWrite"] = _maxWriteTime.ToString() + "ms, " + _maxWriteCommand.ToString();
                             var maxFlush = physical?.MaxFlushTime ?? -1;
                             if (maxFlush >= 0) ex.Data["Redis-MaxFlush"] = maxFlush.ToString() + "ms, " + (physical?.MaxFlushBytes ?? -1).ToString();
                             if (_maxLockDuration >= 0) ex.Data["Redis-MaxLockDuration"] = _maxLockDuration;
-
+#endif
                             message.SetExceptionAndComplete(ex, this);
                         }
                         else
@@ -974,26 +990,33 @@ internal ValueTask<WriteResult> WriteMessageTakingWriteLockAsync(PhysicalConnect
             {
                 if (releaseLock & token.Success)
                 {
+#if DEBUG
                     RecordLockDuration(lockTaken);
+#endif
                     token.Dispose();
                 }
             }
         }
+#if DEBUG
         private void RecordLockDuration(int lockTaken)
         {
+
             var lockDuration = unchecked(Environment.TickCount - lockTaken);
             if (lockDuration > _maxLockDuration) _maxLockDuration = lockDuration;
         }
         volatile int _maxLockDuration = -1;
+#endif
 
-        private async ValueTask<WriteResult> WriteMessageTakingWriteLockAsync_Awaited(ValueTask<LockToken> pending, PhysicalConnection physical, Message message)
+    private async ValueTask<WriteResult> WriteMessageTakingWriteLockAsync_Awaited(ValueTask<LockToken> pending, PhysicalConnection physical, Message message)
         {
             try
             {
                 using (var token = await pending.ForAwait())
                 {
                     if (!token.Success) return TimedOutBeforeWrite(message);
+#if DEBUG
                     int lockTaken = Environment.TickCount;
+#endif
                     var result = WriteMessageInsideLock(physical, message);
 
                     if (result == WriteResult.Success)
@@ -1004,7 +1027,9 @@ private async ValueTask<WriteResult> WriteMessageTakingWriteLockAsync_Awaited(Va
                     UnmarkActiveMessage(message);
                     physical.SetIdle();
 
+#if DEBUG
                     RecordLockDuration(lockTaken);
+#endif
                     return result;
                 }
             }
@@ -1026,7 +1051,9 @@ private async ValueTask<WriteResult> CompleteWriteAndReleaseLockAsync(LockToken
                     return result;
                 }
                 catch (Exception ex) { return HandleWriteException(message, ex); }
+#if DEBUG
                 finally { RecordLockDuration(lockTaken); }
+#endif
             }
         }
 
diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs
@@ -842,12 +842,18 @@ internal static int WriteRaw(Span<byte> span, long value, bool withLengthPrefix
             return WriteCrlf(span, offset);
         }
 
-        private async ValueTask<WriteResult> FlushAsync_Awaited(PhysicalConnection connection, ValueTask<FlushResult> flush, bool throwOnFailure, int startFlush, long flushBytes)
+        private async ValueTask<WriteResult> FlushAsync_Awaited(PhysicalConnection connection, ValueTask<FlushResult> flush, bool throwOnFailure
+#if DEBUG
+            , int startFlush, long flushBytes
+#endif
+            )
         {
             try
             {
                 await flush.ForAwait();
+#if DEBUG
                 RecordEndFlush(startFlush, flushBytes);
+#endif
                 connection._writeStatus = WriteStatus.Flushed;
                 connection.UpdateLastWriteTime();
                 return WriteResult.Success;
@@ -872,7 +878,9 @@ internal WriteResult FlushSync(bool throwOnFailure, int millisecondsTimeout)
 
             void ThrowTimeout()
             {
+#if DEBUG
                 if (millisecondsTimeout > _maxFlushTime) _maxFlushTime = millisecondsTimeout; // a fair bet even if we didn't measure
+#endif
                 throw new TimeoutException("timeout while synchronously flushing");
             }
         }
@@ -883,12 +891,20 @@ internal ValueTask<WriteResult> FlushAsync(bool throwOnFailure)
             try
             {
                 _writeStatus = WriteStatus.Flushing;
+#if DEBUG
                 int startFlush = Environment.TickCount;
                 long flushBytes = -1;
                 if (_ioPipe is SocketConnection sc) flushBytes = sc.GetCounters().BytesWaitingToBeSent;
+#endif
                 var flush = tmp.FlushAsync();
-                if (!flush.IsCompletedSuccessfully) return FlushAsync_Awaited(this, flush, throwOnFailure, startFlush, flushBytes);
+                if (!flush.IsCompletedSuccessfully) return FlushAsync_Awaited(this, flush, throwOnFailure
+#if DEBUG
+                    , startFlush, flushBytes
+#endif
+                );
+#if DEBUG
                 RecordEndFlush(startFlush, flushBytes);
+#endif
                 _writeStatus = WriteStatus.Flushed;
                 UpdateLastWriteTime();
                 return new ValueTask<WriteResult>(WriteResult.Success);
@@ -899,8 +915,10 @@ internal ValueTask<WriteResult> FlushAsync(bool throwOnFailure)
                 return new ValueTask<WriteResult>(WriteResult.WriteFailure);
             }
         }
+#if DEBUG
         private void RecordEndFlush(int start, long bytes)
         {
+
             var end = Environment.TickCount;
             int taken = unchecked(end - start);
             if (taken > _maxFlushTime)
@@ -913,8 +931,9 @@ private void RecordEndFlush(int start, long bytes)
         private long _maxFlushBytes = -1;
         internal int MaxFlushTime => _maxFlushTime;
         internal long MaxFlushBytes => _maxFlushBytes;
+#endif
 
-        private static readonly ReadOnlyMemory<byte> NullBulkString = Encoding.ASCII.GetBytes("$-1\r\n"), EmptyBulkString = Encoding.ASCII.GetBytes("$0\r\n\r\n");
+    private static readonly ReadOnlyMemory<byte> NullBulkString = Encoding.ASCII.GetBytes("$-1\r\n"), EmptyBulkString = Encoding.ASCII.GetBytes("$0\r\n\r\n");
 
         private static void WriteUnifiedBlob(PipeWriter writer, byte[] value)
         {

Original file line number	Diff line number	Diff line change
`@@ -2152,15 +2152,15 @@ private static async Task<T> ExecuteAsyncImpl_Awaited<T>(ConnectionMultiplexer @`
`2152`	`2152`	`return tcs == null ? default(T) : await tcs.Task.ForAwait();`
`2153`	`2153`	`}`
`2154`	`2154`
`2155`		`- internal Exception GetException(WriteResult result, Message message, ServerEndPoint server, [CallerMemberName] string caller = null)`
	`2155`	`+ internal Exception GetException(WriteResult result, Message message, ServerEndPoint server)`
`2156`	`2156`	`{`
`2157`	`2157`	`switch (result)`
`2158`	`2158`	`{`
`2159`	`2159`	`case WriteResult.Success: return null;`
`2160`	`2160`	`case WriteResult.NoConnectionAvailable:`
`2161`	`2161`	`return ExceptionFactory.NoConnectionAvailable(IncludeDetailInExceptions, IncludePerformanceCountersInExceptions, message.Command, message, server, GetServerSnapshot());`
`2162`	`2162`	`case WriteResult.TimeoutBeforeWrite:`
`2163`		`- return ExceptionFactory.Timeout(this, "The timeout was reached before the message could be written to the output buffer, and it was not sent", message, server, result, origin: caller);`
	`2163`	`+ return ExceptionFactory.Timeout(this, "The timeout was reached before the message could be written to the output buffer, and it was not sent", message, server, result);`
`2164`	`2164`	`case WriteResult.WriteFailure:`
`2165`	`2165`	`default:`
`2166`	`2166`	`return ExceptionFactory.ConnectionFailure(IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, "An unknown error occurred when writing the message", server);`
Original file line number	Diff line number	Diff line change
`@@ -181,7 +181,7 @@ internal static string GetLibVersion()`
`181`	`181`	`}`
`182`	`182`	`return _libVersion;`
`183`	`183`	`}`
`184`		`- internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null, string origin = null)`
	`184`	`+ internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult? result = null)`
`185`	`185`	`{`
`186`	`186`	`List<Tuple<string, string>> data = new List<Tuple<string, string>> { Tuple.Create("Message", message.CommandAndKey) };`
`187`	`187`	`var sb = new StringBuilder();`
`@@ -203,16 +203,16 @@ void add(string lk, string sk, string v)`
`203`	`203`	`}`
`204`	`204`	`}`
`205`	`205`
`206`		`- if (!string.IsNullOrWhiteSpace(origin)) add("Origin", null, origin);`
`207`		`-`
`208`	`206`	`// Add timeout data, if we have it`
`209`	`207`	`if (result == WriteResult.TimeoutBeforeWrite)`
`210`	`208`	`{`
`211`	`209`	`add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));`
`212`	`210`	`try`
`213`	`211`	`{`
	`212`	`+#if DEBUG`
`214`	`213`	`if (message.QueuePosition >= 0) add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue`
`215`	`214`	`if ((int)message.ConnectionWriteState >= 0) add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue`
	`215`	`+#endif`
`216`	`216`	`if (message.TryGetPhysicalState(out var state, out var sentDelta, out var receivedDelta))`
`217`	`217`	`{`
`218`	`218`	`add("PhysicalState", "phys", state.ToString());`
Original file line number	Diff line number	Diff line change
`@@ -643,8 +643,10 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message`
`643`	`643`	`Multiplexer?.OnInfoMessage($"reentrant call to WriteMessageTakingWriteLock for {message.CommandAndKey}, {existingMessage.CommandAndKey} is still active");`
`644`	`644`	`return WriteResult.NoConnectionAvailable;`
`645`	`645`	`}`
	`646`	`+#if DEBUG`
`646`	`647`	`int startWriteTime = Environment.TickCount;`
`647`	`648`	`try`
	`649`	`+#endif`
`648`	`650`	`{`
`649`	`651`	`physical.SetWriting();`
`650`	`652`	`var messageIsSent = false;`
`@@ -679,6 +681,7 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message`
`679`	`681`	`return WriteMessageToServerInsideWriteLock(physical, message);`
`680`	`682`	`}`
`681`	`683`	`}`
	`684`	`+#if DEBUG`
`682`	`685`	`finally`
`683`	`686`	`{`
`684`	`687`	`int endWriteTime = Environment.TickCount;`
`@@ -689,10 +692,12 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message`
`689`	`692`	`_maxWriteCommand = message?.Command ?? default;`
`690`	`693`	`}`
`691`	`694`	`}`
	`695`	`+#endif`
`692`	`696`	`}`
`693`		`-`
	`697`	`+#if DEBUG`
`694`	`698`	`private volatile int _maxWriteTime = -1;`
`695`	`699`	`private RedisCommand _maxWriteCommand;`
	`700`	`+#endif`
`696`	`701`
`697`	`702`	`[Obsolete("prefer async")]`
`698`	`703`	`internal WriteResult WriteMessageTakingWriteLockSync(PhysicalConnection physical, Message message)`
`@@ -755,10 +760,14 @@ private bool PushToBacklog(Message message, bool onlyIfExists)`
`755`	`760`	`private void StartBacklogProcessor()`
`756`	`761`	`{`
`757`	`762`	`var sched = Multiplexer.SocketManager?.SchedulerPool ?? DedicatedThreadPoolPipeScheduler.Default;`
	`763`	`+#if DEBUG`
`758`	`764`	`_backlogProcessorRequestedTime = Environment.TickCount;`
	`765`	`+#endif`
`759`	`766`	`sched.Schedule(s_ProcessBacklog, _weakRefThis);`
`760`	`767`	`}`
	`768`	`+#if DEBUG`
`761`	`769`	`private volatile int _backlogProcessorRequestedTime;`
	`770`	`+#endif`
`762`	`771`
`763`	`772`	`private static readonly Action<object> s_ProcessBacklog = s =>`
`764`	`773`	`{`
`@@ -807,20 +816,26 @@ private void ProcessBacklog()`
`807`	`816`	`LockToken token = default;`
`808`	`817`	`try`
`809`	`818`	`{`
	`819`	`+#if DEBUG`
`810`	`820`	`int tryToAcquireTime = Environment.TickCount;`
`811`	`821`	`var msToStartWorker = unchecked(tryToAcquireTime - _backlogProcessorRequestedTime);`
`812`	`822`	`int failureCount = 0;`
	`823`	`+#endif`
`813`	`824`	`while(true)`
`814`	`825`	`{`
`815`	`826`	`// try and get the lock; if unsuccessful, check for termination`
`816`	`827`	`token = _singleWriterMutex.TryWait();`
`817`	`828`	`if (token) break; // got the lock`
`818`	`829`	`lock (_backlog) { if (_backlog.Count == 0) return; }`
	`830`	`+#if DEBUG`
`819`	`831`	`failureCount++;`
	`832`	`+#endif`
`820`	`833`	`}`
`821`	`834`	`_backlogStatus = BacklogStatus.Started;`
	`835`	`+#if DEBUG`
`822`	`836`	`int acquiredTime = Environment.TickCount;`
`823`	`837`	`var msToGetLock = unchecked(acquiredTime - tryToAcquireTime);`
	`838`	`+#endif`
`824`	`839`
`825`	`840`	`// so now we are the writer; write some things!`
`826`	`841`	`Message message;`
`@@ -841,14 +856,15 @@ private void ProcessBacklog()`
`841`	`856`	`{`
`842`	`857`	`_backlogStatus = BacklogStatus.RecordingTimeout;`
`843`	`858`	`var ex = Multiplexer.GetException(WriteResult.TimeoutBeforeWrite, message, ServerEndPoint);`
	`859`	`+#if DEBUG // additional tracking`
`844`	`860`	`ex.Data["Redis-BacklogStartDelay"] = msToStartWorker;`
`845`	`861`	`ex.Data["Redis-BacklogGetLockDelay"] = msToGetLock;`
`846`	`862`	`if (failureCount != 0) ex.Data["Redis-BacklogFailCount"] = failureCount;`
`847`	`863`	`if (_maxWriteTime >= 0) ex.Data["Redis-MaxWrite"] = _maxWriteTime.ToString() + "ms, " + _maxWriteCommand.ToString();`
`848`	`864`	`var maxFlush = physical?.MaxFlushTime ?? -1;`
`849`	`865`	`if (maxFlush >= 0) ex.Data["Redis-MaxFlush"] = maxFlush.ToString() + "ms, " + (physical?.MaxFlushBytes ?? -1).ToString();`
`850`	`866`	`if (_maxLockDuration >= 0) ex.Data["Redis-MaxLockDuration"] = _maxLockDuration;`
`851`		`-`
	`867`	`+#endif`
`852`	`868`	`message.SetExceptionAndComplete(ex, this);`
`853`	`869`	`}`
`854`	`870`	`else`
`@@ -974,26 +990,33 @@ internal ValueTask<WriteResult> WriteMessageTakingWriteLockAsync(PhysicalConnect`
`974`	`990`	`{`
`975`	`991`	`if (releaseLock & token.Success)`
`976`	`992`	`{`
	`993`	`+#if DEBUG`
`977`	`994`	`RecordLockDuration(lockTaken);`
	`995`	`+#endif`
`978`	`996`	`token.Dispose();`
`979`	`997`	`}`
`980`	`998`	`}`
`981`	`999`	`}`
	`1000`	`+#if DEBUG`
`982`	`1001`	`private void RecordLockDuration(int lockTaken)`
`983`	`1002`	`{`
	`1003`	`+`
`984`	`1004`	`var lockDuration = unchecked(Environment.TickCount - lockTaken);`
`985`	`1005`	`if (lockDuration > _maxLockDuration) _maxLockDuration = lockDuration;`
`986`	`1006`	`}`
`987`	`1007`	`volatile int _maxLockDuration = -1;`
	`1008`	`+#endif`
`988`	`1009`
`989`		`- private async ValueTask<WriteResult> WriteMessageTakingWriteLockAsync_Awaited(ValueTask<LockToken> pending, PhysicalConnection physical, Message message)`
	`1010`	`+ private async ValueTask<WriteResult> WriteMessageTakingWriteLockAsync_Awaited(ValueTask<LockToken> pending, PhysicalConnection physical, Message message)`
`990`	`1011`	`{`
`991`	`1012`	`try`
`992`	`1013`	`{`
`993`	`1014`	`using (var token = await pending.ForAwait())`
`994`	`1015`	`{`
`995`	`1016`	`if (!token.Success) return TimedOutBeforeWrite(message);`
	`1017`	`+#if DEBUG`
`996`	`1018`	`int lockTaken = Environment.TickCount;`
	`1019`	`+#endif`
`997`	`1020`	`var result = WriteMessageInsideLock(physical, message);`
`998`	`1021`
`999`	`1022`	`if (result == WriteResult.Success)`
`@@ -1004,7 +1027,9 @@ private async ValueTask<WriteResult> WriteMessageTakingWriteLockAsync_Awaited(Va`
`1004`	`1027`	`UnmarkActiveMessage(message);`
`1005`	`1028`	`physical.SetIdle();`
`1006`	`1029`
	`1030`	`+#if DEBUG`
`1007`	`1031`	`RecordLockDuration(lockTaken);`
	`1032`	`+#endif`
`1008`	`1033`	`return result;`
`1009`	`1034`	`}`
`1010`	`1035`	`}`
`@@ -1026,7 +1051,9 @@ private async ValueTask<WriteResult> CompleteWriteAndReleaseLockAsync(LockToken`
`1026`	`1051`	`return result;`
`1027`	`1052`	`}`
`1028`	`1053`	`catch (Exception ex) { return HandleWriteException(message, ex); }`
	`1054`	`+#if DEBUG`
`1029`	`1055`	`finally { RecordLockDuration(lockTaken); }`
	`1056`	`+#endif`
`1030`	`1057`	`}`
`1031`	`1058`	`}`
`1032`	`1059`