Enhance logging and JSON parsing capabilities

Updated logging configuration in `Program.cs` to clear existing providers and set a new console format with UTC timestamps. Improved structured logging in `LogSummaryHourlyBackgroundService.cs`. Enhanced `LogParser.cs` to support JSON parsing for timestamps and log levels, and refactored the `ParseContainerLogFormat` method for better message extraction. Added new methods for parsing JSON timestamps and log levels.

Author: Timothy Dodd

src/LogMkAgent/Program.cs

@@ -50,11 +50,13 @@ public static async Task Main(string[] args)
                 // Configure logging
                 services.AddLogging(logging =>
                 {
+                    logging.ClearProviders();
                     logging.AddSimpleConsole(options =>
                     {
                         options.SingleLine = true;
-                        options.IncludeScopes = true;
-                        options.TimestampFormat = "HH:mm:ss ";
+                        options.IncludeScopes = false;
+                        options.TimestampFormat = "yyyy-MM-dd HH:mm:ss ";
+                        options.UseUtcTimestamp = true;
                     });
                 });
 

src/LogMkApi/Program.cs

@@ -25,6 +25,17 @@ public class Program
     public static void Main(string[] args)
     {
         var builder = WebApplication.CreateBuilder(args);
+        
+        // Configure cleaner logging format
+        builder.Logging.ClearProviders();
+        builder.Logging.AddSimpleConsole(options =>
+        {
+            options.SingleLine = true;
+            options.TimestampFormat = "yyyy-MM-dd HH:mm:ss ";
+            options.UseUtcTimestamp = true;
+            options.IncludeScopes = false;
+        });
+        
         builder.Services.AddRequestDecompression();
         builder.Services.AddResponseCompression(options =>
         {

src/LogMkApi/Services/LogSummaryHourlyBackgroundService.cs

@@ -35,7 +35,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 var nextRun = CalculateNextRunTime(now);
                 var delay = nextRun - now;
 
-                _logger.LogInformation($"Next log summary update scheduled for {nextRun:yyyy-MM-dd HH:mm:ss}");
+                _logger.LogInformation("Next log summary update scheduled for {NextRun}", nextRun.ToString("yyyy-MM-dd HH:mm:ss"));
 
                 // Wait until the next hour
                 await Task.Delay(delay, stoppingToken);

src/LogMkCommon/LogParser.cs

@@ -1,4 +1,5 @@
 using System.Globalization;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 
 namespace LogMkCommon;
@@ -13,19 +14,32 @@ public static class LogParser
         if (string.IsNullOrWhiteSpace(line))
             return null;
 
+        // Check if line contains JSON and try to parse timestamp from it
+        if (line.TrimStart().StartsWith('{') && line.TrimEnd().EndsWith('}'))
+        {
+            var jsonTimestamp = ParseJsonTimestamp(line);
+            if (jsonTimestamp.HasValue)
+                return jsonTimestamp;
+        }
+
         // First try to parse container log format timestamp (at the beginning)
         var firstSpace = line.IndexOf(' ');
         if (firstSpace > 0)
         {
             var timestampStr = line.Substring(0, firstSpace);
-            var containerTimestamp = ParseContainerTimestamp(timestampStr);
-            if (containerTimestamp.HasValue)
-                return containerTimestamp;
+            // Skip malformed or partial timestamps (like "696426698Z")
+            if (timestampStr.Length >= 20 && timestampStr.Contains("T"))
+            {
+                var containerTimestamp = ParseContainerTimestamp(timestampStr);
+                if (containerTimestamp.HasValue)
+                    return containerTimestamp;
+            }
         }
 
         // Try common timestamp patterns anywhere in the line
         var patterns = new[]
         {
+            @"\[(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\]", // [2025-09-03 22:44:07] format
             @"(\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?)", // ISO 8601
             @"(\d{4}/\d{2}/\d{2}\s+\d{2}:\d{2}:\d{2})", // 2024/01/15 12:34:56
             @"(\d{2}/\d{2}/\d{4}\s+\d{2}:\d{2}:\d{2})", // 01/15/2024 12:34:56
@@ -67,6 +81,14 @@ private static LogLevel GetLogLevel(string logLine)
         if (string.IsNullOrEmpty(logLine))
             return LogLevel.Information;
 
+        // Check if line is JSON and try to parse level from it
+        if (logLine.TrimStart().StartsWith('{') && logLine.TrimEnd().EndsWith('}'))
+        {
+            var jsonLevel = ParseJsonLogLevel(logLine);
+            if (jsonLevel != LogLevel.Information) // If we found a specific level in JSON
+                return jsonLevel;
+        }
+
         var upperLine = logLine.ToUpperInvariant();
 
         // Check for common log level patterns with boundaries
@@ -103,7 +125,8 @@ private static bool ContainsLogLevel(string upperLine, params string[] patterns)
 
     public static string ParseContainerLogFormat(string originalLine, string cleanLine)
     {
-        // Container log format: "timestamp stdout/stderr actual-log-message"
+        // Container log format: "timestamp stdout/stderr F/P actual-log-message"
+        // F = full line, P = partial line
         var firstSpace = originalLine.IndexOf(' ');
         if (firstSpace <= 0)
             return cleanLine;
@@ -112,20 +135,44 @@ public static string ParseContainerLogFormat(string originalLine, string cleanLi
         if (secondSpace <= 0)
             return cleanLine;
 
-        var thirdSpace = originalLine.IndexOf(' ', secondSpace + 1);
-        if (thirdSpace <= 0)
-            return cleanLine;
-
         var outType = originalLine.Substring(firstSpace + 1, secondSpace - firstSpace - 1);
         if (outType == "stdout" || outType == "stderr")
         {
-            // Calculate the position adjustment due to ANSI escape sequences removal
-            var lengthDiff = originalLine.Length - cleanLine.Length;
-            var adjustedPosition = thirdSpace + 1 - lengthDiff;
-
-            if (adjustedPosition >= 0 && adjustedPosition < cleanLine.Length)
+            // Check for the F/P flag after stdout/stderr
+            var thirdSpace = originalLine.IndexOf(' ', secondSpace + 1);
+            if (thirdSpace <= 0)
+                return cleanLine;
+            
+            var flag = originalLine.Substring(secondSpace + 1, thirdSpace - secondSpace - 1);
+            if (flag == "F" || flag == "P")
+            {
+                // Find the actual log message after the flag
+                var fourthSpace = originalLine.IndexOf(' ', thirdSpace + 1);
+                if (fourthSpace > 0)
+                {
+                    // We need to find this position in the clean line
+                    // Count how many characters we need to skip in the clean line
+                    var prefixToSkip = originalLine.Substring(0, fourthSpace + 1);
+                    var cleanPrefixToSkip = RemoveANSIEscapeRegex.Replace(prefixToSkip, string.Empty);
+                    
+                    if (cleanPrefixToSkip.Length < cleanLine.Length)
+                    {
+                        return cleanLine.Substring(cleanPrefixToSkip.Length);
+                    }
+                }
+            }
+            else
             {
-                return cleanLine.Substring(adjustedPosition);
+                // Old format without F/P flag
+                var actualMessageStart = thirdSpace + 1;
+                // Calculate how many chars to skip in clean line
+                var prefixToSkip = originalLine.Substring(0, actualMessageStart);
+                var cleanPrefixToSkip = RemoveANSIEscapeRegex.Replace(prefixToSkip, string.Empty);
+                
+                if (cleanPrefixToSkip.Length < cleanLine.Length)
+                {
+                    return cleanLine.Substring(cleanPrefixToSkip.Length);
+                }
             }
         }
 
@@ -170,4 +217,83 @@ public static string RemoveANSIEscapeSequences(string line)
     {
         return RemoveANSIEscapeRegex.Replace(line, string.Empty);
     }
+
+    private static DateTimeOffset? ParseJsonTimestamp(string line)
+    {
+        try
+        {
+            using var doc = JsonDocument.Parse(line);
+            var root = doc.RootElement;
+            
+            // Try common timestamp field names
+            string[] timestampFields = { "ts", "timestamp", "time", "@timestamp", "datetime", "date" };
+            
+            foreach (var field in timestampFields)
+            {
+                if (root.TryGetProperty(field, out var tsElement))
+                {
+                    var tsValue = tsElement.GetString();
+                    if (!string.IsNullOrEmpty(tsValue))
+                    {
+                        if (DateTimeOffset.TryParse(tsValue, out var timestamp))
+                            return timestamp;
+                    }
+                }
+            }
+            
+            // Try parsing numeric timestamp (Unix epoch)
+            if (root.TryGetProperty("timestamp", out var unixElement))
+            {
+                if (unixElement.ValueKind == JsonValueKind.Number)
+                {
+                    var unixTime = unixElement.GetInt64();
+                    return DateTimeOffset.FromUnixTimeSeconds(unixTime);
+                }
+            }
+        }
+        catch
+        {
+            // Not valid JSON or parsing error, ignore
+        }
+        
+        return null;
+    }
+
+    private static LogLevel ParseJsonLogLevel(string line)
+    {
+        try
+        {
+            using var doc = JsonDocument.Parse(line);
+            var root = doc.RootElement;
+            
+            // Try common log level field names
+            string[] levelFields = { "level", "severity", "log_level", "logLevel", "@level" };
+            
+            foreach (var field in levelFields)
+            {
+                if (root.TryGetProperty(field, out var levelElement))
+                {
+                    var levelValue = levelElement.GetString()?.ToUpperInvariant();
+                    if (!string.IsNullOrEmpty(levelValue))
+                    {
+                        return levelValue switch
+                        {
+                            "ERROR" or "ERR" or "FATAL" or "CRITICAL" => LogLevel.Error,
+                            "WARN" or "WARNING" => LogLevel.Warning,
+                            "INFO" or "INFORMATION" => LogLevel.Information,
+                            "DEBUG" or "DBG" => LogLevel.Debug,
+                            "TRACE" or "TRC" or "VERBOSE" => LogLevel.Trace,
+                            _ => LogLevel.Information
+                        };
+                    }
+                }
+            }
+        }
+        catch
+        {
+            // Not valid JSON or parsing error, ignore
+        }
+        
+        return LogLevel.Information;
+    }
 }