Merge pull request #7 from thiagomvas/feat/dashboard

Feat/dashboard

Author: thiagomvas

docker-compose.dev.yml

@@ -1,44 +1,41 @@
+
 version: "3.9"
 
 services:
   postgres:
     image: postgres:15
     container_name: logport-postgres
-    restart: unless-stopped
-    labels:
-      - com.logport.monitor=true
     environment:
-      POSTGRES_USER: ${LOGPORT_POSTGRES_USER}
-      POSTGRES_PASSWORD: ${LOGPORT_POSTGRES_PASSWORD}
-      POSTGRES_DB: ${LOGPORT_POSTGRES_DB}
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: logport
     ports:
       - "5432:5432"
-    volumes:
-      - pgdata:/var/lib/postgresql/data
-
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1
-    container_name: logport-elasticsearch
-    labels:
-      - com.logport.monitor=true
-    environment:
-      - discovery.type=single-node
-      - ES_JAVA_OPTS=-Xms512m -Xmx512m
-      - xpack.security.enabled=false
-    ports:
-      - "9200:9200"
     networks:
       - logport-network
     volumes:
-      - esdata:/usr/share/elasticsearch/data
+      - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "curl -s http://localhost:9200 || exit 1"]
-      interval: 10s
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
       retries: 5
 
+  logport:
+    image: logport
+    container_name: logport-agent
+    env_file: .env
+    depends_on:
+      postgres:
+        condition: service_healthy
+    networks:
+      - logport-network
+    ports:
+      - "8080:8080"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    user: root
 networks:
   logport-network:
 
 volumes:
   pgdata:
-  esdata:

src/ConsoleApp/Program.cs

@@ -14,7 +14,7 @@
     "Timeout while calling external service"
 };
 
-using var client = LogPortClient.FromServerUrl("ws://localhost:8080/stream");
+using var client = LogPortClient.FromServerUrl("ws://localhost:8080/");
 await client.EnsureConnectedAsync();
 
 // Send a log every time a key is pressed

src/LogPort.Agent/Dockerfile

@@ -4,6 +4,13 @@ WORKDIR /app
 EXPOSE 8080
 EXPOSE 8081
 
+FROM node:20 AS ui-build
+WORKDIR /src/LogPort.UI
+COPY src/LogPort.UI/package*.json ./
+RUN npm ci --verbose
+COPY src/LogPort.UI .
+RUN npm run build
+
 FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
@@ -24,4 +31,5 @@ RUN dotnet publish "./LogPort.Agent.csproj" -c $BUILD_CONFIGURATION -o /app/publ
 FROM base AS final
 WORKDIR /app
 COPY --from=publish /app/publish .
-ENTRYPOINT ["dotnet", "LogPort.Agent.dll"]
+COPY --from=ui-build /src/LogPort.UI/dist ./wwwroot
+ENTRYPOINT ["dotnet", "LogPort.Agent.dll"]

src/LogPort.Agent/Endpoints/AnalyticsEndpoints.cs

@@ -9,12 +9,12 @@ public static class AnalyticsEndpoints
 {
     public static void MapAnalyticsEndpoints(this WebApplication app)
     {
-        app.MapGet("/analytics/histogram", GetHistogram)
+        app.MapGet("api/analytics/histogram", GetHistogram)
         .WithTags("Analytics")
         .WithName("GetLogHistogram")
         .WithSummary("Retrieves a histogram of log entries over time based on the provided query parameters.");
 
-        app.MapGet("/analytics/count-by-type", GetCountByType);
+        app.MapGet("api/analytics/count-by-type", GetCountByType);
     }
 
     private static async Task<IResult> GetHistogram(AnalyticsService service, [AsParameters] LogQueryParameters parameters, [FromQuery] TimeSpan? interval)

src/LogPort.Agent/Endpoints/LogEndpoints.cs

@@ -13,17 +13,19 @@ public static class LogEndpoints
 {
     public static void MapLogEndpoints(this WebApplication app)
     {
-        app.MapPost("/logs", AddLogAsync);
+        app.MapPost("api/logs", AddLogAsync);
 
-        app.MapGet("/logs", GetLogsAsync);
+        app.MapGet("api/logs", GetLogsAsync);
+        app.MapGet("api/logs/count", CountLogsAsync);
+        app.MapGet("api/logs/metadata", GetLogMetadataAsync);
 
         MapStreamEndpoint(app);
         MapLiveLogsEndpoint(app);
     }
 
     private static void MapStreamEndpoint(WebApplication app)
     {
-        app.Map("/stream", async context =>
+        app.Map("api/stream", async context =>
         {
             if (!context.WebSockets.IsWebSocketRequest)
             {
@@ -69,7 +71,7 @@ await webSocket.CloseAsync(System.Net.WebSockets.WebSocketCloseStatus.NormalClos
 
     private static void MapLiveLogsEndpoint(WebApplication app)
     {
-        app.Map("/live-logs", async context =>
+        app.Map("api/live-logs", async context =>
         {
             if (!context.WebSockets.IsWebSocketRequest)
             {
@@ -102,7 +104,7 @@ await webSocket.CloseAsync(System.Net.WebSockets.WebSocketCloseStatus.NormalClos
     private static async Task<IResult> AddLogAsync(ILogRepository logRepository, LogEntry log)
     {
         await logRepository.AddLogAsync(log);
-        return Results.Created($"/logs", log);
+        return Results.Created($"api/logs", log);
     }
 
     private static async Task<IResult> GetLogsAsync(
@@ -112,4 +114,17 @@ private static async Task<IResult> GetLogsAsync(
         var logs = await logRepository.GetLogsAsync(parameters);
         return Results.Ok(logs);
     }
+    private static async Task<IResult> CountLogsAsync(
+        ILogRepository logRepository,
+        [AsParameters] LogQueryParameters parameters)
+    {
+        var count = await logRepository.CountLogsAsync(parameters);
+        return Results.Ok(new { Count = count });
+    }
+
+    private static async Task<IResult> GetLogMetadataAsync(ILogRepository repository)
+    {
+        var metadata = await repository.GetLogMetadataAsync();
+        return Results.Ok(metadata);
+    }
 }

src/LogPort.Agent/Program.cs

@@ -17,7 +17,16 @@
 var builder = WebApplication.CreateBuilder(args);
 
 builder.Services.AddOpenApi();
-
+builder.Services.AddCors(options =>
+{
+    options.AddPolicy("AllowAll", policy =>
+    {
+        policy
+            .AllowAnyOrigin()
+            .AllowAnyMethod()
+            .AllowAnyHeader();
+    });
+});
 builder.Configuration.AddEnvironmentVariables(prefix: "LOGPORT_");
 var logPortConfig = LogPortConfig.LoadFromEnvironment();
 builder.Configuration.GetSection("LOGPORT").Bind(logPortConfig);
@@ -59,11 +68,14 @@
 
 var app = builder.Build();
 
+app.UseCors("AllowAll");
+
 if (app.Environment.IsDevelopment())
 {
     app.MapOpenApi();
 }
 
+
 app.MapHealthChecks("/health", new HealthCheckOptions
 {
     ResponseWriter = async (context, report) =>
@@ -91,6 +103,10 @@
     }
 });
 app.UseWebSockets();
+
+app.UseDefaultFiles();
+app.UseStaticFiles();
+
 app.MapLogEndpoints();
 app.MapAnalyticsEndpoints();
 

src/LogPort.Core/Models/LogMetadata.cs

@@ -0,0 +1,14 @@
+namespace LogPort.Core.Models;
+
+public class LogMetadata
+{
+    public string[] LogLevels { get; set; } = [];
+    public string[] Environments { get; set; } = [];
+    public string[] Services { get; set; } = [];
+    public string[] Hostnames { get; set; } = [];
+    public Dictionary<string, int> LogCountByLevel { get; set; } = new();
+    public Dictionary<string, int> LogCountByService { get; set; } = new();
+    public Dictionary<string, int> LogCountByEnvironment { get; set; } = new();
+    public Dictionary<string, int> LogCountByHostname { get; set; } = new();
+    public long LogCount { get; set; }
+}

src/LogPort.Internal.Common/Interface/ILogRepository.cs

@@ -9,4 +9,5 @@ public interface ILogRepository
     Task<IEnumerable<LogEntry>> GetLogsAsync(LogQueryParameters parameters);
     IAsyncEnumerable<IReadOnlyList<LogEntry>> GetBatchesAsync(LogQueryParameters parameters, int batchSize);
     Task<long> CountLogsAsync(LogQueryParameters parameters);
+    Task<LogMetadata> GetLogMetadataAsync();
 }

src/LogPort.Internal.ElasticSearch/ElasticLogRepository.cs

@@ -105,6 +105,11 @@ public async Task<long> CountLogsAsync(LogQueryParameters parameters)
         return response.Count;
     }
 
+    public Task<LogMetadata> GetLogMetadataAsync()
+    {
+        throw new NotImplementedException();
+    }
+
     private Func<QueryContainerDescriptor<LogEntry>, QueryContainer> BuildQuery(LogQueryParameters parameters)
     {
         return q =>

src/LogPort.Internal.Postgres/PostgresLogRepository.cs

@@ -52,7 +52,8 @@ public async Task AddLogsAsync(IEnumerable<LogEntry> logs)
                 new NpgsqlParameter($"svc{i}", (object?)log.ServiceName ?? DBNull.Value),
                 new NpgsqlParameter($"lvl{i}", log.Level),
                 new NpgsqlParameter($"msg{i}", log.Message),
-                new NpgsqlParameter($"meta{i}", NpgsqlDbType.Jsonb) { Value = JsonSerializer.Serialize(log.Metadata, _jsonOptions) },
+                new NpgsqlParameter($"meta{i}", NpgsqlDbType.Jsonb)
+                    { Value = JsonSerializer.Serialize(log.Metadata, _jsonOptions) },
                 new NpgsqlParameter($"trace{i}", (object?)log.TraceId ?? DBNull.Value),
                 new NpgsqlParameter($"span{i}", (object?)log.SpanId ?? DBNull.Value),
                 new NpgsqlParameter($"host{i}", (object?)log.Hostname ?? DBNull.Value),
@@ -61,8 +62,9 @@ public async Task AddLogsAsync(IEnumerable<LogEntry> logs)
             i++;
         }
 
-        var sql = "INSERT INTO logs (timestamp, service_name, level, message, metadata, trace_id, span_id, hostname, environment) VALUES "
-                  + string.Join(", ", sqlValues);
+        var sql =
+            "INSERT INTO logs (timestamp, service_name, level, message, metadata, trace_id, span_id, hostname, environment) VALUES "
+            + string.Join(", ", sqlValues);
 
         await using var conn = new NpgsqlConnection(_connectionString);
         await conn.OpenAsync();
@@ -108,7 +110,7 @@ public async IAsyncEnumerable<IReadOnlyList<LogEntry>> GetBatchesAsync(
         var sqlParams = new List<NpgsqlParameter>();
         BuildFilters(sql, sqlParams, parameters);
 
-        sql.Append(" ORDER BY timestamp ASC"); 
+        sql.Append(" ORDER BY timestamp ASC");
 
         await using var conn = new NpgsqlConnection(_connectionString);
         await conn.OpenAsync();
@@ -149,9 +151,84 @@ public async Task<long> CountLogsAsync(LogQueryParameters parameters)
         return (long)await cmd.ExecuteScalarAsync();
     }
 
+public async Task<LogMetadata> GetLogMetadataAsync()
+{
+    const string sql = @"
+WITH
+  lvl_counts AS (
+    SELECT jsonb_object_agg(level, count) AS data
+    FROM (SELECT level, COUNT(*) AS count FROM logs WHERE level IS NOT NULL GROUP BY level) t
+  ),
+  svc_counts AS (
+    SELECT jsonb_object_agg(service_name, count) AS data
+    FROM (SELECT service_name, COUNT(*) AS count FROM logs WHERE service_name IS NOT NULL GROUP BY service_name) t
+  ),
+  env_counts AS (
+    SELECT jsonb_object_agg(environment, count) AS data
+    FROM (SELECT environment, COUNT(*) AS count FROM logs WHERE environment IS NOT NULL GROUP BY environment) t
+  ),
+  host_counts AS (
+    SELECT jsonb_object_agg(hostname, count) AS data
+    FROM (SELECT hostname, COUNT(*) AS count FROM logs WHERE hostname IS NOT NULL GROUP BY hostname) t
+  ),
+  distincts AS (
+    SELECT
+      array_agg(DISTINCT level) AS levels,
+      array_agg(DISTINCT environment) AS environments,
+      array_agg(DISTINCT service_name) AS services,
+      array_agg(DISTINCT hostname) AS hostnames,
+      COUNT(*) AS log_count
+    FROM logs
+  )
+SELECT
+  d.levels,
+  d.environments,
+  d.services,
+  d.hostnames,
+  d.log_count,
+  l.data AS log_count_by_level,
+  s.data AS log_count_by_service,
+  e.data AS log_count_by_environment,
+  h.data AS log_count_by_hostname
+FROM distincts d, lvl_counts l, svc_counts s, env_counts e, host_counts h;
+
+";
+
+    await using var conn = new NpgsqlConnection(_connectionString);
+    await conn.OpenAsync();
+    await using var cmd = new NpgsqlCommand(sql, conn);
+    await using var reader = await cmd.ExecuteReaderAsync();
+
+    if (!await reader.ReadAsync())
+        throw new InvalidOperationException("Failed to read log metadata.");
+
+    return new LogMetadata
+    {
+        LogLevels = reader.IsDBNull(0) ? [] : reader.GetFieldValue<string[]>(0),
+        Environments = reader.IsDBNull(1) ? [] : reader.GetFieldValue<string[]>(1),
+        Services = reader.IsDBNull(2) ? [] : reader.GetFieldValue<string[]>(2),
+        Hostnames = reader.IsDBNull(3) ? [] : reader.GetFieldValue<string[]>(3),
+        LogCount = reader.GetInt64(4),
+        LogCountByLevel = reader.IsDBNull(5)
+            ? new()
+            : JsonSerializer.Deserialize<Dictionary<string, int>>(reader.GetString(5))!,
+        LogCountByService = reader.IsDBNull(6)
+            ? new()
+            : JsonSerializer.Deserialize<Dictionary<string, int>>(reader.GetString(6))!,
+        LogCountByEnvironment = reader.IsDBNull(7)
+            ? new()
+            : JsonSerializer.Deserialize<Dictionary<string, int>>(reader.GetString(7))!,
+        LogCountByHostname = reader.IsDBNull(8)
+            ? new()
+            : JsonSerializer.Deserialize<Dictionary<string, int>>(reader.GetString(8))!
+    };
+}
+
+
     private async Task EnsurePartitionAsync(DateTime timestamp)
     {
-        var startDate = timestamp.Date.AddDays(-((timestamp.Date - DateTime.MinValue.Date).Days % _partitionLengthInDays));
+        var startDate =
+            timestamp.Date.AddDays(-((timestamp.Date - DateTime.MinValue.Date).Days % _partitionLengthInDays));
         var endDate = startDate.AddDays(_partitionLengthInDays);
 
         var partitionName = $"logs_{startDate:yyyy_MM_dd}_{_partitionLengthInDays}d";
@@ -205,6 +282,7 @@ void AddFilter(string column, object? value)
             parameters.Add(new NpgsqlParameter($"p{idx}", query.From.Value));
             idx++;
         }
+
         if (query.To.HasValue)
         {
             sql.Append($" AND timestamp <= @p{idx}");
@@ -224,6 +302,7 @@ void AddFilter(string column, object? value)
                 sql.Append($" AND message ILIKE @p{idx}");
                 parameters.Add(new NpgsqlParameter($"p{idx}", $"%{query.Search}%"));
             }
+
             idx++;
         }
 
@@ -256,4 +335,4 @@ private LogEntry MapReader(NpgsqlDataReader reader) =>
             Hostname = reader.IsDBNull(7) ? null : reader.GetString(7),
             Environment = reader.IsDBNull(8) ? null : reader.GetString(8),
         };
-}
+}