logrotator test tool

Author: BRUNER Patrick

src/LogExpert.Core/Classes/Log/LogfileReader.cs

@@ -331,144 +331,150 @@ public int ShiftBuffers ()
         _logger.Info(CultureInfo.InvariantCulture, "ShiftBuffers() begin for {0}{1}", _fileName, IsMultiFile ? " (MultiFile)" : "");
 
         AcquireBufferListWriterLock();
-        ClearBufferState();
-
-        var offset = 0;
-        _isLineCountDirty = true;
 
-        lock (_monitor)
+        try
         {
-            RolloverFilenameHandler rolloverHandler = new(_watchedILogFileInfo, _multiFileOptions);
-            var fileNameList = rolloverHandler.GetNameList(_pluginRegistry);
+            ClearBufferState();
 
-            ResetBufferCache();
+            var offset = 0;
+            _isLineCountDirty = true;
 
-            IList<ILogFileInfo> lostILogFileInfoList = [];
-            IList<ILogFileInfo> readNewILogFileInfoList = [];
-            IList<ILogFileInfo> newFileInfoList = [];
+            lock (_monitor)
+            {
+                RolloverFilenameHandler rolloverHandler = new(_watchedILogFileInfo, _multiFileOptions);
+                var fileNameList = rolloverHandler.GetNameList(_pluginRegistry);
 
-            var enumerator = _logFileInfoList.GetEnumerator();
+                ResetBufferCache();
 
-            while (enumerator.MoveNext())
-            {
-                var logFileInfo = enumerator.Current;
-                var fileName = logFileInfo.FullName;
-                _logger.Debug(CultureInfo.InvariantCulture, "Testing file {0}", fileName);
+                IList<ILogFileInfo> lostILogFileInfoList = [];
+                IList<ILogFileInfo> readNewILogFileInfoList = [];
+                IList<ILogFileInfo> newFileInfoList = [];
 
-                var node = fileNameList.Find(fileName);
+                var enumerator = _logFileInfoList.GetEnumerator();
 
-                if (node == null)
+                while (enumerator.MoveNext())
                 {
-                    _logger.Warn(CultureInfo.InvariantCulture, "File {0} not found", fileName);
-                    continue;
-                }
+                    var logFileInfo = enumerator.Current;
+                    var fileName = logFileInfo.FullName;
+                    _logger.Debug(CultureInfo.InvariantCulture, "Testing file {0}", fileName);
 
-                if (node.Previous != null)
-                {
-                    fileName = node.Previous.Value;
-                    var newILogFileInfo = GetLogFileInfo(fileName);
-                    _logger.Debug(CultureInfo.InvariantCulture, "{0} exists\r\nOld size={1}, new size={2}", fileName, logFileInfo.OriginalLength, newILogFileInfo.Length);
-                    // is the new file the same as the old buffer info?
-                    if (newILogFileInfo.Length == logFileInfo.OriginalLength)
+                    var node = fileNameList.Find(fileName);
+
+                    if (node == null)
                     {
-                        ReplaceBufferInfos(logFileInfo, newILogFileInfo);
-                        newFileInfoList.Add(newILogFileInfo);
+                        _logger.Warn(CultureInfo.InvariantCulture, "File {0} not found", fileName);
+                        continue;
                     }
-                    else
+
+                    if (node.Previous != null)
                     {
-                        _logger.Debug(CultureInfo.InvariantCulture, "Buffer for {0} must be re-read.", fileName);
-                        // not the same. so must read the rest of the list anew from the files
-                        readNewILogFileInfoList.Add(newILogFileInfo);
-                        while (enumerator.MoveNext())
+                        fileName = node.Previous.Value;
+                        var newILogFileInfo = GetLogFileInfo(fileName);
+                        _logger.Debug(CultureInfo.InvariantCulture, "{0} exists\r\nOld size={1}, new size={2}", fileName, logFileInfo.OriginalLength, newILogFileInfo.Length);
+                        // is the new file the same as the old buffer info?
+                        if (newILogFileInfo.Length == logFileInfo.OriginalLength)
                         {
-                            fileName = enumerator.Current.FullName;
-                            node = fileNameList.Find(fileName);
-                            if (node == null)
-                            {
-                                _logger.Warn(CultureInfo.InvariantCulture, "File {0} not found", fileName);
-                                continue;
-                            }
-
-                            if (node.Previous != null)
-                            {
-                                fileName = node.Previous.Value;
-                                _logger.Debug(CultureInfo.InvariantCulture, "New name is {0}", fileName);
-                                readNewILogFileInfoList.Add(GetLogFileInfo(fileName));
-                            }
-                            else
+                            ReplaceBufferInfos(logFileInfo, newILogFileInfo);
+                            newFileInfoList.Add(newILogFileInfo);
+                        }
+                        else
+                        {
+                            _logger.Debug(CultureInfo.InvariantCulture, "Buffer for {0} must be re-read.", fileName);
+                            // not the same. so must read the rest of the list anew from the files
+                            readNewILogFileInfoList.Add(newILogFileInfo);
+                            while (enumerator.MoveNext())
                             {
-                                _logger.Warn(CultureInfo.InvariantCulture, "No previous file for {0} found", fileName);
+                                fileName = enumerator.Current.FullName;
+                                node = fileNameList.Find(fileName);
+                                if (node == null)
+                                {
+                                    _logger.Warn(CultureInfo.InvariantCulture, "File {0} not found", fileName);
+                                    continue;
+                                }
+
+                                if (node.Previous != null)
+                                {
+                                    fileName = node.Previous.Value;
+                                    _logger.Debug(CultureInfo.InvariantCulture, "New name is {0}", fileName);
+                                    readNewILogFileInfoList.Add(GetLogFileInfo(fileName));
+                                }
+                                else
+                                {
+                                    _logger.Warn(CultureInfo.InvariantCulture, "No previous file for {0} found", fileName);
+                                }
                             }
                         }
                     }
+                    else
+                    {
+                        _logger.Info(CultureInfo.InvariantCulture, "{0} does not exist", fileName);
+                        lostILogFileInfoList.Add(logFileInfo);
+                    }
                 }
-                else
+
+                if (lostILogFileInfoList.Count > 0)
                 {
-                    _logger.Info(CultureInfo.InvariantCulture, "{0} does not exist", fileName);
-                    lostILogFileInfoList.Add(logFileInfo);
-                }
-            }
+                    _logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for lost files");
 
-            if (lostILogFileInfoList.Count > 0)
-            {
-                _logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for lost files");
+                    foreach (var logFileInfo in lostILogFileInfoList)
+                    {
+                        var lastDeletedBufferInfo = DeleteBuffersForInfo(logFileInfo, false);
+                        if (lastDeletedBufferInfo != null)
+                        {
+                            offset += lastDeletedBufferInfo.Value.StartLine + lastDeletedBufferInfo.Value.LineCount;
+                        }
+                    }
 
-                foreach (var logFileInfo in lostILogFileInfoList)
-                {
-                    var lastDeletedBufferInfo = DeleteBuffersForInfo(logFileInfo, false);
-                    if (lastDeletedBufferInfo != null)
+                    _logger.Info(CultureInfo.InvariantCulture, "Adjusting StartLine values in {0} buffers by offset {1}", _bufferList.Count, offset);
+                    foreach (var buffer in _bufferList.Values.ToList())
                     {
-                        offset += lastDeletedBufferInfo.Value.StartLine + lastDeletedBufferInfo.Value.LineCount;
+                        SetNewStartLineForBuffer(buffer, buffer.StartLine - offset);
                     }
-                }
 
-                _logger.Info(CultureInfo.InvariantCulture, "Adjusting StartLine values in {0} buffers by offset {1}", _bufferList.Count, offset);
-                foreach (var buffer in _bufferList.Values)
-                {
-                    SetNewStartLineForBuffer(buffer, buffer.StartLine - offset);
+#if DEBUG
+                    if (_bufferList.Values.Count > 0)
+                    {
+                        _logger.Debug(CultureInfo.InvariantCulture, "First buffer now has StartLine {0}", _bufferList.Values[0].StartLine);
+                    }
+#endif
                 }
 
-#if DEBUG
-                if (_bufferList.Values.Count > 0)
+                // Read anew all buffers following a buffer info that couldn't be matched with the corresponding existing file
+                _logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for files that must be re-read");
+
+                foreach (var iLogFileInfo in readNewILogFileInfoList)
                 {
-                    _logger.Debug(CultureInfo.InvariantCulture, "First buffer now has StartLine {0}", _bufferList.Values[0].StartLine);
+                    DeleteBuffersForInfo(iLogFileInfo, true);
                 }
-#endif
-            }
 
-            // Read anew all buffers following a buffer info that couldn't be matched with the corresponding existing file
-            _logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for files that must be re-read");
+                _logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for the watched file");
 
-            foreach (var iLogFileInfo in readNewILogFileInfoList)
-            {
-                DeleteBuffersForInfo(iLogFileInfo, true);
-            }
+                DeleteBuffersForInfo(_watchedILogFileInfo, true);
 
-            _logger.Info(CultureInfo.InvariantCulture, "Deleting buffers for the watched file");
+                _logger.Info(CultureInfo.InvariantCulture, "Re-Reading files");
 
-            DeleteBuffersForInfo(_watchedILogFileInfo, true);
+                foreach (var iLogFileInfo in readNewILogFileInfoList)
+                {
+                    ReadToBufferList(iLogFileInfo, 0, LineCount);
+                    newFileInfoList.Add(iLogFileInfo);
+                }
 
-            _logger.Info(CultureInfo.InvariantCulture, "Re-Reading files");
+                _logFileInfoList = newFileInfoList;
+                _watchedILogFileInfo = GetLogFileInfo(_watchedILogFileInfo.FullName);
+                _logFileInfoList.Add(_watchedILogFileInfo);
+                _logger.Info(CultureInfo.InvariantCulture, "Reading watched file");
 
-            foreach (var iLogFileInfo in readNewILogFileInfoList)
-            {
-                ReadToBufferList(iLogFileInfo, 0, LineCount);
-                newFileInfoList.Add(iLogFileInfo);
+                ReadToBufferList(_watchedILogFileInfo, 0, LineCount);
             }
 
-            _logFileInfoList = newFileInfoList;
-            _watchedILogFileInfo = GetLogFileInfo(_watchedILogFileInfo.FullName);
-            _logFileInfoList.Add(_watchedILogFileInfo);
-            _logger.Info(CultureInfo.InvariantCulture, "Reading watched file");
+            _logger.Info(CultureInfo.InvariantCulture, "ShiftBuffers() end. offset={0}", offset);
 
-            ReadToBufferList(_watchedILogFileInfo, 0, LineCount);
+            return offset;
+        }
+        finally
+        {
+            ReleaseBufferListWriterLock();
         }
-
-        _logger.Info(CultureInfo.InvariantCulture, "ShiftBuffers() end. offset={0}", offset);
-
-        ReleaseBufferListWriterLock();
-
-        return offset;
     }
 
     /// <summary>

src/LogExpert.Tests/BufferShiftTest.cs

@@ -27,6 +27,9 @@ public void Boot ()
     [Test]
     [TestCase(ReaderType.System)]
     //[TestCase(ReaderType.Legacy)] Legacy Reader does not Support this
+    //TO Test real life scenario, use the LogRotator tool, in the src/Tools/LogRotator directory,
+    //to create files and perform rollovers while watching the files in LogExpert with MultiFile enabled
+    //(pattern: *$J(.))
     public void TestShiftBuffers1 (ReaderType readerType)
     {
         var linesPerFile = 10;
@@ -118,7 +121,6 @@ public void TestShiftBuffers1 (ReaderType readerType)
             _ = enumerator.MoveNext();
         }
 
-        _ = enumerator.MoveNext();
         // the last 2 files now contain the content of the previously watched file
         for (; i < logBuffers.Count; ++i)
         {

src/LogExpert.Tests/RolloverHandlerTestBase.cs

@@ -63,7 +63,7 @@ protected static LinkedList<string> RolloverSimulation (LinkedList<string> files
             _ = enumerator.MoveNext();
         }
 
-        _ = CreateFile(null, nextEnumerator.Current);
+        _ = CreateFile(null, enumerator.Current);
 
         if (deleteLatestFile)
         {

src/tools/LogRotator/LogRotator.cs

@@ -0,0 +1,111 @@
+using System.Text;
+
+const string baseDir = "logs";
+const string baseName = "engine.log";
+const int maxBackups = 6;
+const int linesPerFile = 50;
+
+Directory.CreateDirectory(baseDir);
+
+// Create initial set of files
+Console.WriteLine($"Creating initial files in '{Path.GetFullPath(baseDir)}'...");
+WriteLogFile(Path.Combine(baseDir, baseName), 0);
+
+for (var i = 1; i <= maxBackups; i++)
+{
+    WriteLogFile(Path.Combine(baseDir, $"{baseName}.{i}"), i);
+}
+
+PrintFiles();
+Console.WriteLine();
+Console.WriteLine("Open the 'engine.log' file in LogExpert with MultiFile enabled (pattern: *$J(.))");
+Console.WriteLine("Press ENTER to perform a rotation (with oldest file deletion), or Q to quit.");
+
+var rotationCount = 0;
+
+while (true)
+{
+    var key = Console.ReadKey(true);
+
+    if (key.Key == ConsoleKey.Q)
+    {
+        break;
+    }
+
+    if (key.Key != ConsoleKey.Enter)
+    {
+        continue;
+    }
+
+    rotationCount++;
+    Console.WriteLine($"\n--- Rotation #{rotationCount} ---");
+
+    // Delete the oldest file (simulates maxBackups limit)
+    var oldest = Path.Combine(baseDir, $"{baseName}.{maxBackups}");
+
+    if (File.Exists(oldest))
+    {
+        File.Delete(oldest);
+        Console.WriteLine($"  Deleted: {baseName}.{maxBackups}");
+    }
+
+    // Shift all numbered files up by one
+    for (var i = maxBackups - 1; i >= 1; i--)
+    {
+        var src = Path.Combine(baseDir, $"{baseName}.{i}");
+        var dst = Path.Combine(baseDir, $"{baseName}.{i + 1}");
+
+        if (File.Exists(src))
+        {
+            File.Move(src, dst);
+            Console.WriteLine($"  Renamed: {baseName}.{i} -> {baseName}.{i + 1}");
+        }
+    }
+
+    // Rename current log to .1
+    var current = Path.Combine(baseDir, baseName);
+    var first = Path.Combine(baseDir, $"{baseName}.1");
+
+    if (File.Exists(current))
+    {
+        File.Move(current, first);
+        Console.WriteLine($"  Renamed: {baseName} -> {baseName}.1");
+    }
+
+    // Create empty file first (like real log frameworks do), so LogExpert detects
+    // newSize < oldSize and triggers ShiftBuffers()
+    File.Create(current).Dispose();
+    Console.WriteLine($"  Created: {baseName} (empty - triggers rollover detection)");
+
+    PrintFiles();
+
+    // Wait for LogExpert's poll interval to detect the smaller file, then write content
+    Console.WriteLine("  Waiting 2s for LogExpert to detect rollover...");
+    Thread.Sleep(2000);
+
+    WriteLogFile(current, maxBackups + rotationCount);
+    Console.WriteLine($"  Wrote content to {baseName}");
+
+    PrintFiles();
+    Console.WriteLine("\nPress ENTER for next rotation, Q to quit.");
+}
+
+static void WriteLogFile(string path, int fileId)
+{
+    using var writer = new StreamWriter(path, false, Encoding.UTF8);
+
+    for (var i = 1; i <= linesPerFile; i++)
+    {
+        writer.WriteLine($"{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff} [INFO] File#{fileId:D3} Line {i:D3} - {Path.GetFileName(path)} - Sample log message");
+    }
+}
+
+static void PrintFiles()
+{
+    Console.WriteLine("\nCurrent files on disk:");
+
+    foreach (var f in Directory.GetFiles(baseDir, $"{baseName}*").OrderBy(f => f))
+    {
+        Console.WriteLine($"  {Path.GetFileName(f)} ({new FileInfo(f).Length} bytes)");
+    }
+}

src/tools/LogRotator/LogRotator.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+</Project>