Skip to content

Commit 191c4f1

Browse files
authored
Merge pull request #567 from LogExperts/logfilereaderOptimisations
Implement performance optimizations and caching improvements
2 parents f782bae + fa1c5b8 commit 191c4f1

28 files changed

Lines changed: 1252 additions & 962 deletions

build/_build.csproj

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,20 @@
1010
</PropertyGroup>
1111

1212
<ItemGroup>
13-
<PackageReference Include="chocolatey" Version="2.5.1" />
14-
<PackageReference Include="GitVersion.Core" Version="6.5.1" />
15-
<PackageReference Include="Microsoft.Build" Version="18.0.2" />
16-
<PackageReference Include="Microsoft.Build.Tasks.Core" Version="18.0.2" />
17-
<PackageReference Include="NuGet.CommandLine" Version="7.0.3">
13+
<PackageReference Include="chocolatey" Version="2.7.1" />
14+
<PackageReference Include="GitVersion.Core" Version="6.7.0" />
15+
<PackageReference Include="Microsoft.Build" Version="18.4.0" />
16+
<PackageReference Include="Microsoft.Build.Tasks.Core" Version="18.4.0" />
17+
<PackageReference Include="NuGet.CommandLine" Version="7.3.1">
1818
<PrivateAssets>all</PrivateAssets>
1919
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
2020
</PackageReference>
21-
<PackageReference Include="NuGet.Versioning" Version="7.0.1" />
22-
<PackageReference Include="Nuke.Common" Version="10.0.0" />
21+
<PackageReference Include="NuGet.Packaging" Version="7.3.1" />
22+
<PackageReference Include="NuGet.Versioning" Version="7.3.1" />
23+
<PackageReference Include="Nuke.Common" Version="10.1.0" />
2324
<PackageReference Include="Nuke.GitHub" Version="7.0.0" />
24-
<PackageReference Include="NUnit.ConsoleRunner" Version="3.20.2" />
25+
<PackageReference Include="NUnit.ConsoleRunner" Version="3.22.0" />
26+
<PackageReference Include="System.Security.Cryptography.Xml" Version="10.0.6" />
2527
</ItemGroup>
2628

2729
<ItemGroup>

global.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"sdk": {
3-
"version": "10.0.200",
3+
"version": "10.0.100",
44
"rollForward": "latestPatch"
55
}
66
}

src/ColumnizerLib/LogLine.cs

Lines changed: 12 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,19 @@ namespace ColumnizerLib;
44
/// Represents a single log line, including its full text and line number.
55
/// </summary>
66
/// <remarks>
7-
/// <para>
8-
/// <b>Purpose:</b> <br/>
9-
/// The <c>LogLine</c> struct encapsulates the content and line number of a log entry. It is used throughout the
10-
/// columnizer and log processing infrastructure to provide a strongly-typed, immutable representation of a log line.
11-
/// </para>
12-
/// <para>
13-
/// <b>Usage:</b> <br/>
14-
/// This struct implements the <see cref="ILogLineMemory"/> interface, allowing it to be used wherever an <c>ILogLineMemory</c>
15-
/// is expected. It provides value semantics and is intended to be lightweight and efficiently passed by value.
16-
/// </para>
17-
/// <para>
18-
/// <b>Relationship to ILogLineMemory:</b> <br/>
19-
/// <c>LogLine</c> is a concrete, immutable implementation of the <see cref="ILogLineMemory"/> interface, providing
20-
/// properties for the full line text and its line number.
21-
/// </para>
22-
/// <para>
23-
/// <b>Why struct instead of record:</b> <br/>
24-
/// A <c>struct</c> is preferred over a <c>record</c> here to avoid heap allocations and to provide value-type
25-
/// semantics, which are beneficial for performance when processing large numbers of log lines. The struct is
26-
/// immutable (readonly), ensuring thread safety and predictability. The previous <c>record</c> implementation
27-
/// was replaced to better align with these performance and semantic requirements.
28-
/// </para>
7+
/// <para> <b> Purpose:</b> <br/> The <c> LogLine</c> struct encapsulates the content and line number of a log entry. It
8+
/// is used throughout the columnizer and log processing infrastructure to provide a strongly-typed, immutable
9+
/// representation of a log line. </para>
10+
/// <para> <b> Usage:</b> <br/> This struct implements the
11+
/// <see cref="ILogLineMemory"/> interface, allowing it to be used wherever an <c> ILogLineMemory</c> is expected. It
12+
/// provides value semantics and is intended to be lightweight and efficiently passed by value. </para> <para> <b>
13+
/// Relationship to ILogLineMemory:</b> <br/> <c> LogLine</c> is a concrete, immutable implementation of the
14+
/// <see cref="ILogLineMemory"/> interface, providing properties for the full line text and its line number. </para>
15+
/// This is a readonly record struct implementing
16+
/// <see cref="ILogLineMemory"/>. Stored inline in <c> List&lt;LogLine&gt;</c> to avoid boxing and heap allocation.
17+
/// Boxing occurs only when returned through the <c> ILogLineMemory</c> interface boundary.
2918
/// </remarks>
30-
public class LogLine : ILogLineMemory
19+
public readonly record struct LogLine : ILogLineMemory
3120
{
3221
public int LineNumber { get; }
3322

src/Directory.Build.props

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
<Project>
22
<PropertyGroup>
3-
<Version>1.31.0.0</Version>
4-
<AssemblyVersion>1.31.0.0</AssemblyVersion>
5-
<FileVersion>1.31.0.0</FileVersion>
6-
<InformationalVersion>1.31.0.0</InformationalVersion>
3+
<Version>1.40.0.0</Version>
4+
<AssemblyVersion>1.40.0.0</AssemblyVersion>
5+
<FileVersion>1.40.0.0</FileVersion>
6+
<InformationalVersion>1.40.0.0</InformationalVersion>
77
<Authors>Hirogen, zarunbal, RandallFlagg, TheNicker</Authors>
88
<Company>LogExperts</Company>
99
<ImplicitUsings>enable</ImplicitUsings>
@@ -21,8 +21,8 @@
2121
<RepositoryUrl>https://github.com/LogExperts/LogExpert</RepositoryUrl>
2222
<PackageTags>LogExpert, Columnizer, Logging, Windows, Winforms</PackageTags>
2323
<RepositoryType>git</RepositoryType>
24-
<PackageReleaseNotes>https://github.com/LogExperts/LogExpert/releases/tag/v.1.31.0</PackageReleaseNotes>
25-
<PackageVersion>1.31.0.0</PackageVersion>
24+
<PackageReleaseNotes>https://github.com/LogExperts/LogExpert/releases/tag/v.1.40.0</PackageReleaseNotes>
25+
<PackageVersion>1.40.0.0</PackageVersion>
2626
<Optimize Condition="'$(Configuration)' == 'Release'">true</Optimize>
2727
<Product>LogExpert</Product>
2828
<Copyright>Copyright © LogExpert 2025</Copyright>

src/Directory.Packages.props

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,24 +10,24 @@
1010
<PackageVersion Include="CsvHelper" Version="33.1.0" />
1111
<PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.1" />
1212
<PackageVersion Include="GitVersion.Core" Version="6.3.0" />
13-
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
13+
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.4.0" />
1414
<PackageVersion Include="Moq" Version="4.20.72" />
1515
<PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
16-
<PackageVersion Include="NLog" Version="6.0.6" />
16+
<PackageVersion Include="NLog" Version="6.1.2" />
1717
<PackageVersion Include="NuGet.CommandLine" Version="6.14.0" />
18-
<PackageVersion Include="NuGet.Versioning" Version="7.0.1" />
18+
<PackageVersion Include="NuGet.Versioning" Version="7.3.1" />
1919
<PackageVersion Include="Nuke.Common" Version="9.0.4" />
2020
<PackageVersion Include="Nuke.GitHub" Version="7.0.0" />
21-
<PackageVersion Include="NUnit" Version="4.4.0" />
21+
<PackageVersion Include="NUnit" Version="4.5.1" />
2222
<PackageVersion Include="NUnit.ConsoleRunner" Version="3.20.1" />
23-
<PackageVersion Include="NUnit3TestAdapter" Version="5.2.0" />
23+
<PackageVersion Include="NUnit3TestAdapter" Version="6.2.0" />
2424
<PackageVersion Include="SSH.NET" Version="2025.1.0" />
25-
<PackageVersion Include="System.Drawing.Common" Version="10.0.0" />
26-
<PackageVersion Include="System.Resources.Extensions" Version="10.0.0" />
27-
<PackageVersion Include="Vanara.Library" Version="4.2.1" />
28-
<PackageVersion Include="Vanara.PInvoke.DwmApi" Version="4.2.1" />
29-
<PackageVersion Include="Vanara.PInvoke.RstrtMgr" Version="4.2.1" />
30-
<PackageVersion Include="Vanara.PInvoke.Shell32" Version="4.2.1" />
31-
<PackageVersion Include="Vanara.PInvoke.User32" Version="4.2.1" />
25+
<PackageVersion Include="System.Drawing.Common" Version="10.0.6" />
26+
<PackageVersion Include="System.Resources.Extensions" Version="10.0.6" />
27+
<PackageVersion Include="Vanara.Library" Version="5.0.4" />
28+
<PackageVersion Include="Vanara.PInvoke.DwmApi" Version="5.0.4" />
29+
<PackageVersion Include="Vanara.PInvoke.RstrtMgr" Version="5.0.4" />
30+
<PackageVersion Include="Vanara.PInvoke.Shell32" Version="5.0.4" />
31+
<PackageVersion Include="Vanara.PInvoke.User32" Version="5.0.4" />
3232
</ItemGroup>
3333
</Project>
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
namespace LogExpert.Core.Classes.Log;
2+
3+
/// <summary>
4+
/// Stores byte offsets for each line start in a file. Supports incremental appending for tail mode.
5+
/// </summary>
6+
internal sealed class LineOffsetIndex (int initialCapacity = 4096)
7+
{
8+
private long[] _offsets = new long[initialCapacity];
9+
10+
public int LineCount { get; private set; }
11+
12+
/// <summary>
13+
/// Appends a line-start offset.
14+
/// </summary>
15+
public void Add (long offset)
16+
{
17+
if (LineCount == _offsets.Length)
18+
{
19+
Array.Resize(ref _offsets, _offsets.Length * 2);
20+
}
21+
22+
_offsets[LineCount++] = offset;
23+
}
24+
25+
/// <summary>
26+
/// Returns the byte offset of the start of the given line.
27+
/// </summary>
28+
public long GetOffset (int lineNum)
29+
{
30+
return (uint)lineNum < (uint)LineCount ? _offsets[lineNum] : -1;
31+
}
32+
33+
/// <summary>
34+
/// Returns the byte length of the given line (from its start to the next line's start).
35+
/// For the last line, returns -1 (unknown length, read to end or newline).
36+
/// </summary>
37+
public long GetLineLength (int lineNum)
38+
{
39+
return (uint)lineNum >= (uint)LineCount
40+
? -1
41+
: lineNum + 1 < LineCount
42+
? _offsets[lineNum + 1] - _offsets[lineNum]
43+
: -1;
44+
}
45+
46+
/// <summary>
47+
/// Removes all offsets, resetting the index.
48+
/// </summary>
49+
public void Clear ()
50+
{
51+
LineCount = 0;
52+
}
53+
}

src/LogExpert.Core/Classes/Log/LogBuffer.cs

Lines changed: 78 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
using System.Buffers;
2+
13
using ColumnizerLib;
24

35
using NLog;
@@ -8,29 +10,33 @@ public class LogBuffer
810
{
911
#region Fields
1012

13+
private SpinLock _contentLock = new(enableThreadOwnerTracking: false);
1114
private static readonly Logger _logger = LogManager.GetCurrentClassLogger();
1215

1316
#if DEBUG
14-
private readonly IList<long> _filePositions = []; // file position for every line
17+
private readonly List<long> _filePositions; // file position for every line
1518
#endif
1619

17-
private readonly List<ILogLineMemory> _lineList = [];
20+
private LogLine[] _lineArray;
21+
private int _lineArrayLength; // capacity of the rented array
1822

1923
private int MAX_LINES = 500;
20-
private long _size;
2124

2225
#endregion
2326

2427
#region cTor
2528

26-
//public LogBuffer() { }
27-
2829
// Don't use a primary constructor here: field initializers (like MAX_LINES) run before primary constructor parameters are assigned,
2930
// so MAX_LINES would always be set to its default value before the constructor body can assign it. Use a regular constructor instead.
3031
public LogBuffer (ILogFileInfo fileInfo, int maxLines)
3132
{
3233
FileInfo = fileInfo;
3334
MAX_LINES = maxLines;
35+
_lineArray = ArrayPool<LogLine>.Shared.Rent(maxLines);
36+
_lineArrayLength = _lineArray.Length;
37+
#if DEBUG
38+
_filePositions = new(MAX_LINES);
39+
#endif
3440
}
3541

3642
#endregion
@@ -43,18 +49,18 @@ public long Size
4349
{
4450
set
4551
{
46-
_size = value;
52+
field = value;
4753
#if DEBUG
4854
if (_filePositions.Count > 0)
4955
{
50-
if (_size < _filePositions[_filePositions.Count - 1] - StartPos)
56+
if (field < _filePositions[^1] - StartPos)
5157
{
5258
_logger.Error("LogBuffer overall Size must be greater than last line file position!");
5359
}
5460
}
5561
#endif
5662
}
57-
get => _size;
63+
get;
5864
}
5965

6066
public int EndLine => StartLine + LineCount;
@@ -75,36 +81,91 @@ public long Size
7581

7682
#region Public methods
7783

78-
public void AddLine (ILogLineMemory lineMemory, long filePos)
84+
public void AddLine (LogLine lineMemory, long filePos)
7985
{
80-
_lineList.Add(lineMemory);
86+
if (LineCount < _lineArrayLength)
87+
{
88+
_lineArray[LineCount] = lineMemory;
89+
LineCount++;
90+
}
91+
#if DEBUG
92+
else
93+
{
94+
_logger.Error("AddLine overflow: LineCount={0} >= _lineArrayLength={1}", LineCount, _lineArrayLength);
95+
}
96+
#endif
97+
8198
#if DEBUG
8299
_filePositions.Add(filePos);
83100
#endif
84-
LineCount++;
85101
IsDisposed = false;
86102
}
87103

88104
public void ClearLines ()
89105
{
90-
_lineList.Clear();
106+
Array.Clear(_lineArray, 0, LineCount);
91107
LineCount = 0;
92108
}
93109

110+
/// <summary>
111+
/// Prepares the buffer for reuse from the pool.
112+
/// </summary>
113+
public void Reinitialise (ILogFileInfo fileInfo, int maxLines)
114+
{
115+
FileInfo = fileInfo;
116+
MAX_LINES = maxLines;
117+
StartLine = 0;
118+
StartPos = 0;
119+
Size = 0;
120+
LineCount = 0;
121+
DroppedLinesCount = 0;
122+
PrevBuffersDroppedLinesSum = 0;
123+
IsDisposed = false;
124+
_lineArray = ArrayPool<LogLine>.Shared.Rent(maxLines);
125+
_lineArrayLength = _lineArray.Length;
126+
#if DEBUG
127+
_filePositions.Clear();
128+
DisposeCount = 0;
129+
#endif
130+
}
131+
94132
public void DisposeContent ()
95133
{
96-
_lineList.Clear();
134+
if (_lineArray != null)
135+
{
136+
Array.Clear(_lineArray, 0, LineCount);
137+
ArrayPool<LogLine>.Shared.Return(_lineArray);
138+
_lineArray = null;
139+
LineCount = 0;
140+
}
141+
97142
IsDisposed = true;
98143
#if DEBUG
99144
DisposeCount++;
100145
#endif
101146
}
102147

103-
public ILogLineMemory GetLineMemoryOfBlock (int num)
148+
public LogLine? GetLineMemoryOfBlock (int num)
149+
{
150+
return num < LineCount && num >= 0
151+
? _lineArray[num]
152+
: null;
153+
}
154+
155+
/// <summary>
156+
/// Acquires the content lock. The caller MUST call <see cref="ReleaseContentLock"/> in a finally block.
157+
/// </summary>
158+
public void AcquireContentLock (ref bool lockTaken)
159+
{
160+
_contentLock.Enter(ref lockTaken);
161+
}
162+
163+
/// <summary>
164+
/// Releases the content lock previously acquired via <see cref="AcquireContentLock"/>.
165+
/// </summary>
166+
public void ReleaseContentLock ()
104167
{
105-
return num < _lineList.Count && num >= 0
106-
? _lineList[num]
107-
: null;
168+
_contentLock.Exit(useMemoryBarrier: false);
108169
}
109170

110171
#endregion

0 commit comments

Comments
 (0)