diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index 7b7f84c3b..99bc74395 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -1,7 +1,7 @@ /* Technitium DNS Server -Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) -Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) +Copyright (C) 2025 Shreyas Zare +Copyright (C) 2025 Zafer Balkan This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,18 +15,17 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the You should have received a copy of the GNU General Public License along with this program. If not, see . - */ using DnsServerCore.ApplicationCommon; -using LogExporter.Strategy; +using LogExporter.Pipeline; +using LogExporter.Sinks; using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; -using TechnitiumLibrary; using TechnitiumLibrary.Net.Dns; namespace LogExporter @@ -35,181 +34,378 @@ public sealed class App : IDnsApplication, IDnsQueryLogger { #region variables - IDnsServer? _dnsServer; - AppConfig? _config; + private const int BULK_INSERT_COUNT = 1000; + + private readonly SinkDispatcher _sinkDispatcher; + private readonly PipelineDispatcher _enrichmentDispatcher; - readonly ExportManager _exportManager = new ExportManager(); + // Stage 1 buffer: transformed LogEntry waiting for enrichment + private Channel _transformChannel = default!; - bool _enableLogging; + // Stage 2 buffer: enriched LogEntry waiting for dispatch + private Channel _enrichedChannel = default!; - readonly ConcurrentQueue _queuedLogs = new ConcurrentQueue(); - readonly Timer _queueTimer; - const int QUEUE_TIMER_INTERVAL = 10000; - const int BULK_INSERT_COUNT = 1000; + private Task? _backgroundTask; + private AppConfig? _config; + private bool _disposed; + private IDnsServer? _dnsServer; + private volatile bool _enableLogging; // volatile to improve cross-thread visibility - bool _disposed; + private long _droppedCount; + private static readonly TimeSpan DropLogInterval = TimeSpan.FromSeconds(5); + private long _lastDropTicks; - #endregion + #endregion variables #region constructor public App() { - _queueTimer = new Timer(HandleExportLogCallback); + _sinkDispatcher = new SinkDispatcher(); + _enrichmentDispatcher = new PipelineDispatcher(); + _lastDropTicks = DateTime.UtcNow.Ticks; } - #endregion + #endregion constructor #region IDisposable + ~App() => Dispose(); + public void Dispose() { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } + if (_disposed) + return; - private void Dispose(bool disposing) - { - if (!_disposed) + _disposed = true; + + // Stop accepting new entries immediately; cannot throw. + _enableLogging = false; + + // Best-effort shutdown: complete input channel so workers drain and exit. + try { - if (disposing) + try + { + _transformChannel?.Writer.TryComplete(); + } + catch (Exception ex) { - _queueTimer?.Dispose(); + _dnsServer?.WriteLog(ex); + } + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } - ExportLogsAsync().Sync(); //flush any pending logs + // Wait for background pipeline to finish. + try + { + _backgroundTask?.GetAwaiter().GetResult(); + } + catch (OperationCanceledException) + { + // Not expected without explicit cancellation, but safe to ignore. + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } - _exportManager.Dispose(); - } + // Dispose sinks and enrichment dispatcher defensively. + try + { + _sinkDispatcher.Dispose(); + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } - _disposed = true; + try + { + _enrichmentDispatcher.Dispose(); + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); } + + GC.SuppressFinalize(this); } - #endregion + #endregion IDisposable #region public public Task InitializeAsync(IDnsServer dnsServer, string config) { - _dnsServer = dnsServer; - _config = AppConfig.Deserialize(config); + ObjectDisposedException.ThrowIf(_disposed, this); - if (_config is null) - throw new DnsClientException("Invalid application configuration."); + _dnsServer = dnsServer; - if (_config.FileTarget!.Enabled) - { - _exportManager.RemoveStrategy(typeof(FileExportStrategy)); - _exportManager.AddStrategy(new FileExportStrategy(_config.FileTarget!.Path)); - } - else + try { - _exportManager.RemoveStrategy(typeof(FileExportStrategy)); - } + _config = AppConfig.Deserialize(config) + ?? throw new DnsClientException("Invalid application configuration."); - if (_config.HttpTarget!.Enabled) - { - _exportManager.RemoveStrategy(typeof(HttpExportStrategy)); - _exportManager.AddStrategy(new HttpExportStrategy(_config.HttpTarget.Endpoint, _config.HttpTarget.Headers)); + ConfigurePipeline(); + + ConfigureSinks(); } - else + catch (Exception ex) { - _exportManager.RemoveStrategy(typeof(HttpExportStrategy)); + // Fail fast but log with context; do not partially initialize pipeline. + _dnsServer?.WriteLog(ex); + _enableLogging = false; + throw; } - if (_config.SyslogTarget!.Enabled) - { - _exportManager.RemoveStrategy(typeof(SyslogExportStrategy)); - _exportManager.AddStrategy(new SyslogExportStrategy(_config.SyslogTarget.Address, _config.SyslogTarget.Port, _config.SyslogTarget.Protocol)); - } - else + // If no sinks exist, never enable logging. + if (!_sinkDispatcher.Any()) { - _exportManager.RemoveStrategy(typeof(SyslogExportStrategy)); + _enableLogging = false; + return Task.CompletedTask; } - _enableLogging = _exportManager.HasStrategy(); - - if (_enableLogging) - _queueTimer.Change(QUEUE_TIMER_INTERVAL, Timeout.Infinite); - else - _queueTimer.Change(Timeout.Infinite, Timeout.Infinite); + // Stage 1: transform buffer – InsertLogAsync pushes LogEntry here. + _transformChannel = Channel.CreateBounded( + new BoundedChannelOptions(_config!.Sinks.MaxQueueSize) + { + SingleReader = true, + SingleWriter = false, // InsertLogAsync may be called concurrently + FullMode = BoundedChannelFullMode.DropWrite + }); + + // Stage 2: enriched buffer – EnrichLogsAsync pushes here, ExportLogsAsync consumes. + _enrichedChannel = Channel.CreateBounded( + new BoundedChannelOptions(_config.Sinks.MaxQueueSize) + { + SingleReader = true, + SingleWriter = true, // only enrichment stage writes + FullMode = BoundedChannelFullMode.DropWrite + }); + + // Start pipeline workers: + // - EnrichLogsAsync: transform -> enrich + // - ExportLogsAsync: enrich -> output + _backgroundTask = Task.WhenAll( + Task.Run(EnrichLogsAsync), + Task.Run(ExportLogsAsync)); + + // ADR: _enableLogging is intentionally set last so that any caller observing + // _enableLogging is true can rely on the entire logging pipeline being fully + // constructed (channels and background workers). This prevents subtle race + // conditions where concurrent InsertLogAsync calls see "enabled" before internal + // structures are ready. + _enableLogging = true; return Task.CompletedTask; } - public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + // Step 1: input + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, + IPEndPoint remoteEP, DnsTransportProtocol protocol, + DnsDatagram response) { if (_enableLogging) { - if (_queuedLogs.Count < _config!.MaxQueueSize) - _queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging)); + LogEntry entry; + + try + { + // input -> transform: build LogEntry + entry = new LogEntry(timestamp, remoteEP, protocol, request, response, _config!.Sinks.EnableEdnsLogging); + } + catch (Exception ex) + { + // Malformed packet or unexpected data should not crash the server. + _dnsServer?.WriteLog(ex); + return Task.CompletedTask; + } + + try + { + if (!_transformChannel.Writer.TryWrite(entry)) + { + IncrementDropAndMaybeLog(); + } + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } } return Task.CompletedTask; } - #endregion + #endregion public #region private - private async Task ExportLogsAsync() + // Step 2: EnrichLogsAsync – transform -> enrich + private async Task EnrichLogsAsync() { try { - List logs = new List(BULK_INSERT_COUNT); - - while (true) + while (await _transformChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) { - while (logs.Count < BULK_INSERT_COUNT && _queuedLogs.TryDequeue(out LogEntry? log)) + while (_transformChannel.Reader.TryRead(out LogEntry? entry)) { - logs.Add(log); + // If there is no question, most enrichers cannot do anything. + if (entry.Question != null && _enrichmentDispatcher.Any()) + { + try + { + _enrichmentDispatcher.Run(entry, ex => _dnsServer?.WriteLog(ex)); + } + catch (Exception ex) + { + // Extra guard: dispatcher itself should not tear down the loop. + _dnsServer?.WriteLog(ex); + } + } + + try + { + if (!_enrichedChannel.Writer.TryWrite(entry)) + { + IncrementDropAndMaybeLog(); + } + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } } - - if (logs.Count < 1) - break; - - await _exportManager.ImplementStrategyAsync(logs); - - logs.Clear(); } } catch (Exception ex) { _dnsServer?.WriteLog(ex); } + finally + { + // Signal no more enriched entries will be produced. + try + { + _enrichedChannel.Writer.TryComplete(); + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } + } } - private async void HandleExportLogCallback(object? state) + // Step 3: ExportLogsAsync – pipeline -> output + private async Task ExportLogsAsync() { + // ADR: Reuse this list buffer to avoid GC churn during high-volume logging. + List batch = new List(BULK_INSERT_COUNT); + try { - // Process logs within the timer interval, then let the timer reschedule - await ExportLogsAsync(); + while (await _enrichedChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) + { + while (batch.Count < BULK_INSERT_COUNT && + _enrichedChannel.Reader.TryRead(out LogEntry? entry)) + { + batch.Add(entry); + } + + if (batch.Count > 0) + { + try + { + await _sinkDispatcher + .DispatchAsync(batch, CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) + { + // Sink failures must be logged but must not crash the server. + _dnsServer?.WriteLog(ex); + } + finally + { + batch.Clear(); // REUSE — do not reassign + } + } + } } catch (Exception ex) { _dnsServer?.WriteLog(ex); } - finally + } + + private void ConfigureSinks() + { + var sinks = _config!.Sinks; + _sinkDispatcher.Remove(typeof(ConsoleSink)); + if (sinks.ConsoleSinkConfig != null && sinks.ConsoleSinkConfig.Enabled) + _sinkDispatcher.Add(new ConsoleSink()); + + _sinkDispatcher.Remove(typeof(FileSink)); + if (sinks.FileSinkConfig?.Enabled is true) + _sinkDispatcher.Add(new FileSink(sinks.FileSinkConfig.Path)); + + _sinkDispatcher.Remove(typeof(HttpSink)); + if (sinks.HttpSinkConfig?.Enabled is true) { - try - { - _queueTimer?.Change(QUEUE_TIMER_INTERVAL, Timeout.Infinite); - } - catch (ObjectDisposedException) - { } + _sinkDispatcher.Add(new HttpSink(sinks.HttpSinkConfig)); } - } - #endregion + _sinkDispatcher.Remove(typeof(SyslogSink)); + if (sinks.SyslogSinkConfig?.Enabled is true) + { + _sinkDispatcher.Add( + new SyslogSink(sinks.SyslogSinkConfig.Address, + sinks.SyslogSinkConfig.Port!.Value, + sinks.SyslogSinkConfig.Protocol)); + } + } - #region properties + private void ConfigurePipeline() + { + // Remove any existing enricher types first to avoid duplicate registration. + _enrichmentDispatcher.Remove(typeof(Normalize)); + if (_config!.Pipeline.NormalizeProcessConfig?.Enabled is true) + { + _enrichmentDispatcher.Add(new Normalize()); + } + if (_config!.Pipeline.TaggingProcessConfig?.Enabled is true) + { + _enrichmentDispatcher.Add(new Tags(_config.Pipeline.TaggingProcessConfig.Tags)); + } + } - public string Description + private void IncrementDropAndMaybeLog() { - get { return "Allows exporting query logs to third party sinks. It supports exporting to File, HTTP endpoint, and Syslog (UDP, TCP, TLS, and Local protocols)."; } + Interlocked.Increment(ref _droppedCount); + + long nowTicks = DateTime.UtcNow.Ticks; + long lastTicks = Volatile.Read(ref _lastDropTicks); + + if (new TimeSpan(nowTicks - lastTicks) >= DropLogInterval && + Interlocked.CompareExchange(ref _lastDropTicks, nowTicks, lastTicks) == lastTicks) + { + long dropped = Interlocked.Exchange(ref _droppedCount, 0); + _dnsServer?.WriteLog( + $"Log export queue full; dropped {dropped} entries over last {DropLogInterval.TotalSeconds:F0}s."); + } } - #endregion + #endregion private + + #region properties + + public string Description => + "Allows exporting query logs to third party sinks. Supports exporting to FileSink, HTTP endpoint, and SyslogSink."; + + #endregion properties } } diff --git a/Apps/LogExporterApp/AppConfig.cs b/Apps/LogExporterApp/AppConfig.cs index 22dcaaf81..38908e2a9 100644 --- a/Apps/LogExporterApp/AppConfig.cs +++ b/Apps/LogExporterApp/AppConfig.cs @@ -19,69 +19,153 @@ You should have received a copy of the GNU General Public License */ using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Text.Json; using System.Text.Json.Serialization; +using TechnitiumLibrary.Net.Dns; +using static LogExporter.SinkConfig; namespace LogExporter { public class AppConfig { - [JsonPropertyName("maxQueueSize")] - public int MaxQueueSize { get; set; } + [JsonPropertyName("sinks")] + public SinkConfig Sinks { get; set; } + + [JsonPropertyName("pipeline")] + public PipelineConfig Pipeline { get; set; } + + /// + /// Loads config and enforces DataAnnotations validation. + /// + /// ADR: Validation is intentionally centralized here so that: + /// - App receives only a fully valid configuration. + /// - Errors surface early with domain-specific messages. + /// - No runtime failures occur deep inside the logging pipeline. + /// This ensures plugin initialization is deterministic and safe. + /// + /// + public static AppConfig Deserialize(string json) + { + AppConfig config = JsonSerializer.Deserialize(json, DnsConfigSerializerOptions.Default) + ?? throw new DnsClientException("Configuration could not be deserialized."); - [JsonPropertyName("enableEdnsLogging ")] - public bool EnableEdnsLogging { get; set; } + ValidateObject(config); - [JsonPropertyName("file")] - public FileTarget? FileTarget { get; set; } + // Validate enabled targets only — disabled ones may be incomplete by design. - [JsonPropertyName("http")] - public HttpTarget? HttpTarget { get; set; } + if (config.Sinks.FileSinkConfig?.Enabled is true) + ValidateObject(config.Sinks.FileSinkConfig); - [JsonPropertyName("syslog")] - public SyslogTarget? SyslogTarget { get; set; } + if (config.Sinks.HttpSinkConfig?.Enabled is true) + ValidateObject(config.Sinks.HttpSinkConfig); + + if (config.Sinks.SyslogSinkConfig?.Enabled is true) + ValidateObject(config.Sinks.SyslogSinkConfig); - // Load configuration from JSON - public static AppConfig? Deserialize(string json) + return config; + } + + private static void ValidateObject(object instance) { - return JsonSerializer.Deserialize(json, DnsConfigSerializerOptions.Default); + ValidationContext ctx = new ValidationContext(instance); + Validator.ValidateObject(instance, ctx, validateAllProperties: true); } } - - public class TargetBase + public class FeatureBase { [JsonPropertyName("enabled")] - public bool Enabled { get; set; } + public bool Enabled { get; set; } = true; } - public class SyslogTarget : TargetBase + public class SinkConfig { - [JsonPropertyName("address")] - public required string Address { get; set; } + [Range(1, int.MaxValue, ErrorMessage = "maxQueueSize must be greater than zero.")] - [JsonPropertyName("port")] - public int? Port { get; set; } + [JsonPropertyName("maxQueueSize")] + public int MaxQueueSize { get; set; } = int.MaxValue; - [JsonPropertyName("protocol")] - public string? Protocol { get; set; } - } + [JsonPropertyName("enableEdnsLogging")] + public bool EnableEdnsLogging { get; set; } = true; - public class FileTarget : TargetBase - { - [JsonPropertyName("path")] - public required string Path { get; set; } + [JsonPropertyName("console")] + public ConsoleSink ConsoleSinkConfig { get; set; } + + [JsonPropertyName("file")] + public FileSink FileSinkConfig { get; set; } + + [JsonPropertyName("http")] + public HttpSink HttpSinkConfig { get; set; } + + [JsonPropertyName("syslog")] + public SyslogSink SyslogSinkConfig { get; set; } + + public class SyslogSink : FeatureBase + { + [Required(ErrorMessage = "syslog.address is required when syslog logging is enabled.")] + [JsonPropertyName("address")] + public string Address { get; set; } + + [Range(1, 65535)] + [JsonPropertyName("port")] + public int? Port { get; set; } + + [AllowedValues(["UDP", "TCP", "TLS", "LOCAL"])] + [JsonPropertyName("protocol")] + public string Protocol { get; set; } + } + + public class ConsoleSink : FeatureBase + { + } + + public class FileSink : FeatureBase + { + [Required(ErrorMessage = "file.path is required when syslog logging is enabled.")] + [JsonPropertyName("path")] + public string Path { get; set; } + } + + public class HttpSink : FeatureBase + { + + [Required(ErrorMessage = "http.endpoint is required when HTTP logging is enabled.")] + [Url] + [JsonPropertyName("endpoint")] + public string Endpoint { get; set; } + + [JsonPropertyName("headers")] + public Dictionary? Headers { get; set; } + + [JsonPropertyName("ndjson")] + public bool NdJson { get; set; } = true; + } } - public class HttpTarget : TargetBase + public class PipelineConfig { - [JsonPropertyName("endpoint")] - public required string Endpoint { get; set; } + [JsonPropertyName("normalize")] + public NormalizeProcess NormalizeProcessConfig { get; set; } - [JsonPropertyName("headers")] - public Dictionary? Headers { get; set; } - } + [JsonPropertyName("tagging")] + public TaggingProcess TaggingProcessConfig { get; set; } - // Setup reusable options with a single instance + public class NormalizeProcess : FeatureBase + {} + + public class TaggingProcess : FeatureBase + { + [Required(ErrorMessage = "tags are required when tagging is enabled.")] + [JsonPropertyName("tags")] + public List Tags { get; set; } = new List(); + } + + } + /// + /// Shared serializer configuration for reading dnsApp.config. + /// ADR: The serializer options are centralized so that parsing behavior + /// is stable and predictable across the entire plugin lifetime. + /// public static class DnsConfigSerializerOptions { public static readonly JsonSerializerOptions Default = new JsonSerializerOptions diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs index 72fcb2768..22465cadd 100644 --- a/Apps/LogExporterApp/LogEntry.cs +++ b/Apps/LogExporterApp/LogEntry.cs @@ -1,4 +1,4 @@ -/* +/* Technitium DNS Server Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) @@ -14,10 +14,9 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with this program. If not, see . +along with this program. If not, see */ - using DnsServerCore.ApplicationCommon; using System; using System.Collections.Generic; @@ -33,11 +32,18 @@ namespace LogExporter { public class LogEntry { + // Reuse empty lists to avoid allocations when there are no answers or EDNS data + private static readonly DnsResourceRecord[] EmptyAnswers = Array.Empty(); + private static readonly EDNSLog[] EmptyEdns = Array.Empty(); + public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false) { // Assign timestamp and ensure it's in UTC Timestamp = timestamp.Kind == DateTimeKind.Utc ? timestamp : timestamp.ToUniversalTime(); + // Set hostname + NameServer = request.Metadata.NameServer.Host; + // Extract client information ClientIp = remoteEP.Address.ToString(); Protocol = protocol; @@ -61,48 +67,108 @@ public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol pr }; } - // Convert answer section into a simple string summary (comma-separated for multiple answers) - Answers = new List(response.Answer.Count); + // Convert answer section - reuse empty list when no answers if (response.Answer.Count > 0) { - Answers.AddRange(response.Answer.Select(record => new DnsResourceRecord - { - Name = record.Name, - RecordType = record.Type, - RecordClass = record.Class, - RecordTtl = record.TTL, - RecordData = record.RDATA.ToString(), - DnssecStatus = record.DnssecStatus, - })); + Answers = new List( + response.Answer.Select(record => new DnsResourceRecord + { + Name = record.Name, + RecordType = record.Type, + RecordClass = record.Class, + RecordTtl = record.TTL, + RecordData = record.RDATA.ToString(), + DnssecStatus = record.DnssecStatus, + })).ToArray(); } + else + { + Answers = EmptyAnswers; + } + + PopulateEDNSLogs(response, ednsLogging); + } - EDNS = new List(); + private void PopulateEDNSLogs(DnsDatagram response, bool ednsLogging) + { + // Handle EDNS - reuse empty list when no EDNS logging or no errors if (!ednsLogging || response.EDNS is null) { + EDNS = EmptyEdns; + return; + } + + List ednsErrors = response.EDNS.Options.Where(o => o.Code == EDnsOptionCode.EXTENDED_DNS_ERROR).ToList(); + if (ednsErrors.Count == 0) + { + EDNS = EmptyEdns; return; } - foreach (EDnsOption extendedErrorLog in response.EDNS.Options.Where(o => o.Code == EDnsOptionCode.EXTENDED_DNS_ERROR)) + List edns = new List(ednsErrors.Count); + foreach (EDnsOption extendedErrorLog in ednsErrors) { - string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", StringSplitOptions.TrimEntries); + // ADR: EDNS extended error comes from network input and may not follow + // the expected "type: message" format. Previously this code assumed + // a well-formed structure and could throw IndexOutOfRangeException, + // allowing remote parties to crash the logging pipeline. + // We now parse defensively and treat malformed data as a best-effort message. + + string? raw = extendedErrorLog.Data?.ToString(); + if (string.IsNullOrWhiteSpace(raw)) + continue; + + raw = raw.Replace("[", "").Replace("]", ""); + + string? errType = null; + string? message = null; - EDNS.Add(new EDNSLog + string[] parts = raw.Split(':', 2, StringSplitOptions.TrimEntries); + if (parts.Length == 2) { - ErrType = extractedData[0], - Message = extractedData[1] + errType = parts[0]; + message = parts[1]; + } + else + { + // fallback: treat the raw payload as the message + message = raw; + } + + edns.Add(new EDNSLog + { + ErrType = errType, + Message = message }); } + + // If no valid EDNS entries were added, use the empty list + EDNS = edns.Count == 0 ? EmptyEdns : edns.ToArray(); } - public List Answers { get; private set; } - public string ClientIp { get; private set; } - public List EDNS { get; private set; } - public DnsTransportProtocol Protocol { get; private set; } - public DnsQuestion? Question { get; private set; } - public DnsResponseCode ResponseCode { get; private set; } + public DnsResourceRecord[] Answers { get; } + + public string ClientIp { get; } + + public EDNSLog[] EDNS { get; private set; } + + public string NameServer { get; } + + public DnsTransportProtocol Protocol { get; } + + public DnsQuestion? Question { get; } + + public DnsResponseCode ResponseCode { get; } + public double? ResponseRtt { get; private set; } - public DnsServerResponseType ResponseType { get; private set; } - public DateTime Timestamp { get; private set; } + + public DnsServerResponseType ResponseType { get; } + + public DateTime Timestamp { get; } + + // Meta bag populated by pipeline stages + public Dictionary Meta { get; } = new(); + public override string ToString() { return JsonSerializer.Serialize(this, DnsLogSerializerOptions.Default); @@ -124,16 +190,16 @@ public static class DnsLogSerializerOptions public class DnsQuestion { public DnsClass QuestionClass { get; set; } - public required string QuestionName { get; set; } + public string QuestionName { get; set; } public DnsResourceRecordType QuestionType { get; set; } } public class DnsResourceRecord { public DnssecStatus DnssecStatus { get; set; } - public required string Name { get; set; } + public string Name { get; set; } public DnsClass RecordClass { get; set; } - public required string RecordData { get; set; } + public string RecordData { get; set; } public uint RecordTtl { get; set; } public DnsResourceRecordType RecordType { get; set; } } diff --git a/Apps/LogExporterApp/LogExporterApp.csproj b/Apps/LogExporterApp/LogExporterApp.csproj index 789b6ecbf..c92022078 100644 --- a/Apps/LogExporterApp/LogExporterApp.csproj +++ b/Apps/LogExporterApp/LogExporterApp.csproj @@ -4,7 +4,7 @@ net9.0 false true - 2.1 + 3.0 false Technitium Technitium DNS Server @@ -20,10 +20,8 @@ - - - - + + diff --git a/Apps/LogExporterApp/Strategy/IExportStrategy.cs b/Apps/LogExporterApp/Pipeline/IPipelineProcessor.cs similarity index 70% rename from Apps/LogExporterApp/Strategy/IExportStrategy.cs rename to Apps/LogExporterApp/Pipeline/IPipelineProcessor.cs index 32073e78c..d4e9f2069 100644 --- a/Apps/LogExporterApp/Strategy/IExportStrategy.cs +++ b/Apps/LogExporterApp/Pipeline/IPipelineProcessor.cs @@ -1,4 +1,4 @@ -/* +/* Technitium DNS Server Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) @@ -15,20 +15,14 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the You should have received a copy of the GNU General Public License along with this program. If not, see . - */ using System; -using System.Collections.Generic; -using System.Threading.Tasks; -namespace LogExporter.Strategy +namespace LogExporter.Pipeline { - /// - /// Strategy interface to decide the sinks for exporting the logs. - /// - public interface IExportStrategy: IDisposable + public interface IPipelineProcessor : IDisposable { - Task ExportAsync(IReadOnlyList logs); + void Process(LogEntry logEntry); } -} +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Pipeline/Normalize.DomainCache.cs b/Apps/LogExporterApp/Pipeline/Normalize.DomainCache.cs new file mode 100644 index 000000000..aba33bf09 --- /dev/null +++ b/Apps/LogExporterApp/Pipeline/Normalize.DomainCache.cs @@ -0,0 +1,284 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +using Nager.PublicSuffix; +using Nager.PublicSuffix.RuleProviders; +using Nager.PublicSuffix.RuleProviders.CacheProviders; +using System; +using System.Collections.Concurrent; +using System.Net.Http; +using System.Threading; + +namespace LogExporter.Pipeline +{ + + public partial class Normalize + { + /// + /// Thread-safe cache for parsed domain information using the SIEVE eviction algorithm. + /// SIEVE provides better scan resistance than LRU, making it ideal for DNS workloads + /// where one-time queries (typos, DGA domains) are common. + /// + /// Reference: "SIEVE is Simpler than LRU: an Efficient Turn-Key Eviction Algorithm for + /// Web Caches" (NSDI '24) + /// + internal sealed class DomainCache + { + #region variables + + private const int MaxSize = 10000; + private const int StringPoolMaxSize = 10000; + private static readonly DomainInfo Empty = new DomainInfo(); + + // ADR: Loading the PSL must not block or fail plugin startup. We defer + // initialization and make it best-effort to avoid network dependencies. + private static readonly Lazy _parser = new Lazy(InitializeParser); + + private static readonly HttpClient _pslHttpClient = new HttpClient(); + + private static readonly Lazy _sharedRuleProvider = + new Lazy(static () => + { + LocalFileSystemCacheProvider cacheProvider = new LocalFileSystemCacheProvider(); + CachedHttpRuleProvider rp = new CachedHttpRuleProvider(cacheProvider, _pslHttpClient); + rp.BuildAsync().GetAwaiter().GetResult(); + return rp; + }, isThreadSafe: true); + + private readonly ConcurrentDictionary _cache = + new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _stringPool = + new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private readonly Lock _evictionLock = new Lock(); + + // SIEVE data structures + private CacheNode? _head; + private CacheNode? _tail; + private CacheNode? _hand; + #endregion + + #region public + public DomainInfo GetOrAdd(string domainName) + { + if (string.IsNullOrWhiteSpace(domainName)) + return Empty; + + // Fast path: try cache lookup with original name first (case-insensitive) + if (_cache.TryGetValue(domainName, out CacheNode? node)) + { + node.Visited = true; + return node.Domain; + } + + // NormalizeConfig only if needed, using string pool to reduce allocations + string normalizedName = GetPooledNormalizedName(domainName); + + // Check cache again with normalized name (may differ from original) + if (!ReferenceEquals(normalizedName, domainName) && + _cache.TryGetValue(normalizedName, out node)) + { + node.Visited = true; + return node.Domain; + } + + DomainInfo domain = Parse(domainName); + AddToCache(normalizedName, domain); + return domain; + } + + + public void Clear() + { + lock (_evictionLock) + { + _cache.Clear(); + _stringPool.Clear(); + _head = null; + _tail = null; + _hand = null; + } + } + + #endregion + + #region private + + /// + /// Returns a pooled, normalized version of the domain name to reduce allocations. + /// If the name is already normalized, returns the original string. + /// + private string GetPooledNormalizedName(string name) + { + if (!NeedsNormalization(name)) + return name; + + string normalized = name.ToLowerInvariant().TrimEnd('.'); + + // Try to get from pool, or add if not present + if (_stringPool.TryGetValue(normalized, out string? pooled)) + return pooled; + + // Limit pool size to prevent unbounded growth + if (_stringPool.Count < StringPoolMaxSize) + { + _stringPool.TryAdd(normalized, normalized); + } + + return normalized; + } + + /// + /// Checks if the domain name needs normalization (has uppercase or trailing dot). + /// + private static bool NeedsNormalization(string name) + { + if (name.Length > 0 && name[^1] == '.') + return true; + + foreach (char c in name) + { + if (c >= 'A' && c <= 'Z') + return true; + } + + return false; + } + + private static DomainInfo Parse(string name) + { + DomainParser? parser = _parser.Value; + if (parser == null) + return Empty; + + try + { + return parser.Parse(name) ?? Empty; + } + catch + { + // Parsing errors are intentionally ignored because PSL is optional. + return Empty; + } + } + + private static DomainParser? InitializeParser() + { + // ADR: The PSL download via SimpleHttpRuleProvider performs outbound HTTP. + // Relying on external network connectivity at plugin startup is unsafe in + // production DNS environments (offline appliances, firewalled networks, + // corporate proxies). Initialization must never block or fail due to PSL + // retrieval. We therefore treat PSL availability as optional: + // - If the download succeeds, domain parsing is enriched. + // - If it fails, we return null and logging continues without PSL data. + try + { + return new DomainParser(_sharedRuleProvider.Value); + } + catch + { + return null; + } + } + + private void AddToCache(string key, DomainInfo domain) + { + lock (_evictionLock) + { + if (_cache.ContainsKey(key)) + return; + + while (_cache.Count >= MaxSize) + Evict(); + + CacheNode newNode = new CacheNode(key, domain); + InsertAtHead(newNode); + _cache[key] = newNode; + } + } + + private void InsertAtHead(CacheNode node) + { + node.Next = _head; + node.Prev = null; + + if (_head != null) + _head.Prev = node; + + _head = node; + + _tail ??= node; + + _hand ??= node; + } + + private void Evict() + { + _hand ??= _tail; + + while (_hand != null) + { + if (!_hand.Visited) + { + CacheNode victim = _hand; + _hand = _hand.Prev ?? _tail; + RemoveNode(victim); + _cache.TryRemove(victim.Key, out _); + return; + } + + _hand.Visited = false; + _hand = _hand.Prev ?? _tail; + } + } + + private void RemoveNode(CacheNode node) + { + if (node.Prev != null) + node.Prev.Next = node.Next; + else + _head = node.Next; + + if (node.Next != null) + node.Next.Prev = node.Prev; + else + _tail = node.Prev; + + if (_hand == node) + _hand = node.Prev ?? _tail; + } + + private class CacheNode + { + public readonly string Key; + public readonly DomainInfo Domain; + public volatile bool Visited; + public CacheNode? Next; + public CacheNode? Prev; + + public CacheNode(string key, DomainInfo domain) + { + Key = key; + Domain = domain; + } + } + #endregion + } + } + +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Pipeline/Normalize.cs b/Apps/LogExporterApp/Pipeline/Normalize.cs new file mode 100644 index 000000000..ba103d01d --- /dev/null +++ b/Apps/LogExporterApp/Pipeline/Normalize.cs @@ -0,0 +1,44 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +using System; + +namespace LogExporter.Pipeline +{ + public partial class Normalize : IPipelineProcessor + { + private static readonly DomainCache _domainCache = new DomainCache(); + + public void Process(LogEntry logEntry) + { + if (logEntry.Question == null) + return; + + // store under a well-known key you can standardize keys if you like + logEntry.Meta["domainInfo"] = _domainCache.GetOrAdd(logEntry.Question.QuestionName); + } + + public void Dispose() + { + // If DomainCache ever needs disposal, do it here. + GC.SuppressFinalize(this); + } + } + +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Pipeline/PipelineDispatcher.cs b/Apps/LogExporterApp/Pipeline/PipelineDispatcher.cs new file mode 100644 index 000000000..83db46370 --- /dev/null +++ b/Apps/LogExporterApp/Pipeline/PipelineDispatcher.cs @@ -0,0 +1,147 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare +Copyright (C) 2025 Zafer Balkan + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +using System; +using System.Collections.Concurrent; + +namespace LogExporter.Pipeline +{ + /// + /// Dispatches pipeline actions to all configured IPipelineProcessor strategies. + /// + /// ADR: Meta is synchronous and in-process, so this dispatcher + /// executes strategies sequentially to keep ordering deterministic. + /// Each processor is isolated with its own exception boundary so that + /// one faulty processor cannot break the pipeline. + /// + public sealed class PipelineDispatcher : IDisposable + { + #region variables + + private readonly ConcurrentDictionary _processors = + new ConcurrentDictionary(); + + private bool _disposed; + + #endregion + + #region IDisposable + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + + foreach (IPipelineProcessor enricher in _processors.Values) + { + try + { + enricher.Dispose(); + } + catch + { + // At this point we cannot rely on any logging infrastructure. + // Best-effort only: swallow to avoid secondary failures. + } + } + + _processors.Clear(); + } + + #endregion + + #region public + + public void Add(IPipelineProcessor processor) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ArgumentNullException.ThrowIfNull(processor); + + _processors.AddOrUpdate( + processor.GetType(), + processor, + (_, existing) => + { + // Replace existing instance defensively. + try + { + existing.Dispose(); + } + catch + { + // Ignore disposal failure; new instance still becomes active. + } + + return processor; + }); + } + + public void Remove(Type type) + { + ObjectDisposedException.ThrowIf(_disposed, this); + ArgumentNullException.ThrowIfNull(type); + + if (_processors.TryRemove(type, out IPipelineProcessor? existing)) + { + try + { + existing.Dispose(); + } + catch + { + // Isolation: disposal of one processor must not affect others. + } + } + } + + public bool Any() + { + if (_disposed) + return false; + + return !_processors.IsEmpty; + } + + /// + /// Runs all configured enrichment strategies on a single log entry. + /// Errors are reported to the optional error callback but never thrown. + /// + public void Run(LogEntry logEntry, Action? onError = null) + { + if (_disposed || logEntry == null || _processors.IsEmpty) + return; + + foreach (IPipelineProcessor processor in _processors.Values) + { + try + { + processor.Process(logEntry); + } + catch (Exception ex) + { + onError?.Invoke(ex); + } + } + } + + #endregion + } +} diff --git a/Apps/LogExporterApp/Pipeline/Tags.cs b/Apps/LogExporterApp/Pipeline/Tags.cs new file mode 100644 index 000000000..3ff3e32a6 --- /dev/null +++ b/Apps/LogExporterApp/Pipeline/Tags.cs @@ -0,0 +1,45 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace LogExporter.Pipeline +{ + public partial class Tags : IPipelineProcessor + { + private readonly string[] _tags; + public Tags(IEnumerable tags) + { + _tags = tags.ToArray(); + } + + public void Process(LogEntry logEntry) + { + logEntry.Meta["tags"] = _tags; + } + + public void Dispose() + { + // If DomainCache ever needs disposal, do it here. + GC.SuppressFinalize(this); + } + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/README.md b/Apps/LogExporterApp/README.md new file mode 100644 index 000000000..740ee1330 --- /dev/null +++ b/Apps/LogExporterApp/README.md @@ -0,0 +1,131 @@ +# Log Exporter for Technitium DNS Server + +A plugin that exports DNS query logs to external sinks such as files, HTTP endpoints and Syslog servers. The plugin now supports enrichment stages before export, providing additional derived metadata. + +It maintains an in-memory queue with bulk processing and supports EDNS and optional enrichment layers. + +## Features + +* Captures DNS queries and responses using the Technitium DNS `IDnsQueryLogger` interface. +* Performs enrichment before exporting (currently Public Suffix List resolution). +* Queues log entries asynchronously and flushes them in batches. +* Exports logs via pluggable output sinks: console, file, HTTP POST and Syslog. +* Includes nameserver, question, answer, protocol, RTT, EDNS details and additional enrichment data. +* Prevents unbounded memory usage through bounded log pipelines. +* Flushes pending logs on shutdown. + +## Configuration + +Provide JSON configuration similar to: + +```json +{ + "maxQueueSize": 50000, + "enableEdnsLogging": true, + "enablePslResolution": { + "enabled": true + }, + "console": { + "enabled": true + }, + "file": { + "enabled": false, + "path": "/var/log/technitium/dns.log" + }, + "http": { + "enabled": true, + "endpoint": "https://collector.example.com/dns", + "headers": { "Authorization": "Bearer token" } + }, + "syslog": { + "enabled": false, + "address": "10.0.0.5", + "port": 6514, + "protocol": "tls" + } +} +``` + +## Sample dig result for technitium.com query + +```bash +dig '@127.0.0.1' technitium.com + +; <<>> DiG 9.16.25 <<>> @127.0.0.1 technitium.com +; (1 server found) +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 62685 +;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 + +;; OPT PSEUDOSECTION: +; EDNS: version: 0, flags:; udp: 1232 +;; QUESTION SECTION: +;technitium.com. IN A + +;; ANSWER SECTION: +technitium.com. 8218 IN A 206.189.140.177 + +;; Query time: 24 msec +;; SERVER: 127.0.0.1#53(127.0.0.1) +;; WHEN: Mon Dec 08 22:50:37 FLE Standard Time 2025 +;; MSG SIZE rcvd: 59 + +``` + +## Sample log fot technitium.com query + +Otiginal log: + +```json +{"answers":[{"dnssecStatus":"Disabled","name":"technitium.com","recordClass":"IN","recordData":"206.189.140.177","recordTtl":8218,"recordType":"A"}],"clientIp":"127.0.0.1","edns":[],"nameServer":"127.0.0.1","protocol":"Udp","question":{"questionClass":"IN","questionName":"technitium.com","questionType":"A"},"responseCode":"NoError","responseType":"Cached","timestamp":"2025-12-08T20:50:37.321Z","enrichment":{"domainInfo":{"domain":"technitium","topLevelDomain":"com","registrableDomain":"technitium.com","fullyQualifiedDomainName":"technitium.com","topLevelDomainRule":{"name":"com","type":"Normal","labelCount":1,"division":"ICANN"}}}} +``` + +Formatted for easier review. + +```json +{ + "answers": [ + { + "dnssecStatus": "Disabled", + "name": "technitium.com", + "recordClass": "IN", + "recordData": "206.189.140.177", + "recordTtl": 8218, + "recordType": "A" + } + ], + "clientIp": "127.0.0.1", + "edns": [], + "nameServer": "127.0.0.1", + "protocol": "Udp", + "question": { + "questionClass": "IN", + "questionName": "technitium.com", + "questionType": "A" + }, + "responseCode": "NoError", + "responseType": "Cached", + "timestamp": "2025-12-08T20:50:37.321Z", + "enrichment": { + "domainInfo": { + "domain": "technitium", + "topLevelDomain": "com", + "registrableDomain": "technitium.com", + "fullyQualifiedDomainName": "technitium.com", + "topLevelDomainRule": { + "name": "com", + "type": "Normal", + "labelCount": 1, + "division": "ICANN" + } + } + } +} +``` +## Key notes + +* `enablePslResolution` controls Public Suffix List enrichment. +* Enrichment adds fields under the `enrichment` dictionary inside each log. +* EDNS extended error information is included when enabled. +* `maxQueueSize` applies backpressure across the pipeline. diff --git a/Apps/LogExporterApp/Sinks/ConsoleSink.cs b/Apps/LogExporterApp/Sinks/ConsoleSink.cs new file mode 100644 index 000000000..d445d2a3c --- /dev/null +++ b/Apps/LogExporterApp/Sinks/ConsoleSink.cs @@ -0,0 +1,66 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using Microsoft.IO; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Sinks +{ + public sealed class ConsoleSink : IOutputSink + { + private readonly RecyclableMemoryStreamManager _memoryManager = + new RecyclableMemoryStreamManager(); + + private readonly Stream _stdout; + private bool _disposed; + + public ConsoleSink() + { + _stdout = Console.OpenStandardOutput(); + } + + public void Dispose() + { + // ADR: We intentionally do NOT dispose _stdout. The standard output stream is + // owned by the process, not by this strategy. Disposing it here would break + // all console output for the entire DNS server. Dispose only flips the flag + // so that subsequent ExportAsync calls become no-ops during shutdown. + _disposed = true; + } + + public async Task ExportAsync(IReadOnlyList logs, CancellationToken token) + { + if (_disposed || logs.Count == 0 || token.IsCancellationRequested) + return; + + using RecyclableMemoryStream ms = _memoryManager.GetStream("ConsoleExport-Batch"); + NdjsonSerializer.WriteBatch(ms, logs); + + ms.Position = 0; + + await ms.CopyToAsync(_stdout, token).ConfigureAwait(false); + await _stdout.FlushAsync(token).ConfigureAwait(false); + } + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Sinks/FileSink.cs b/Apps/LogExporterApp/Sinks/FileSink.cs new file mode 100644 index 000000000..cabfabf02 --- /dev/null +++ b/Apps/LogExporterApp/Sinks/FileSink.cs @@ -0,0 +1,92 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using Microsoft.IO; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Sinks +{ + public sealed class FileSink : IOutputSink + { + #region variables + + private readonly FileStream _fileStream; + private readonly RecyclableMemoryStreamManager _memoryManager = new(); + private readonly StreamWriter _writer; + private bool _disposed; + + #endregion variables + + #region constructor + + public FileSink(string filePath) + { + _fileStream = new FileStream( + filePath, + FileMode.Append, + FileAccess.Write, + FileShare.Read, + bufferSize: 64 * 1024, + useAsync: true); + + _writer = new StreamWriter(_fileStream); + } + + #endregion constructor + + #region IDisposable + + public void Dispose() + { + if (_disposed) + return; + + _writer.Dispose(); + _fileStream.Dispose(); + _disposed = true; + } + + #endregion IDisposable + + #region public + + public async Task ExportAsync(IReadOnlyList logs, CancellationToken token) + { + // ADR: FileSink writes must honor cancellation so server shutdown cannot block + // on slow disks or large flush operations. Previously FlushAsync() was not + // cancellable, allowing shutdown to hang indefinitely under I/O pressure. + // All I/O operations now respect the provided token. + if (_disposed || logs.Count == 0 || token.IsCancellationRequested) + return; + + using RecyclableMemoryStream ms = _memoryManager.GetStream("FileExport-Batch"); + NdjsonSerializer.WriteBatch(ms, logs); + ms.Position = 0; + + await ms.CopyToAsync(_writer.BaseStream, token).ConfigureAwait(false); + await _writer.BaseStream.FlushAsync(token).ConfigureAwait(false); + } + + #endregion public + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Sinks/HttpSink.cs b/Apps/LogExporterApp/Sinks/HttpSink.cs new file mode 100644 index 000000000..d61bfc827 --- /dev/null +++ b/Apps/LogExporterApp/Sinks/HttpSink.cs @@ -0,0 +1,176 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using Microsoft.IO; +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using HttpSinkConfig = LogExporter.SinkConfig.HttpSink; + +namespace LogExporter.Sinks +{ + public sealed class HttpSink : IOutputSink + { + #region variables + + private readonly Uri _endpoint; + private readonly HttpClient _httpClient; + private readonly RecyclableMemoryStreamManager _memoryManager = new(); + private readonly bool _ndJson; + private bool _disposed; + + #endregion variables + + #region constructor + + public HttpSink(HttpSinkConfig config) + { + if (!Uri.TryCreate(config.Endpoint, UriKind.Absolute, out Uri? uri)) + throw new ArgumentException("Invalid HTTP endpoint.", nameof(config)); + + _endpoint = uri; + _ndJson = config.NdJson; + _httpClient = new HttpClient(); + + if (config.Headers == null) + { + return; + } + + foreach (var kv in config.Headers.Where(kv => !_httpClient.DefaultRequestHeaders.TryAddWithoutValidation(kv.Key, kv.Value))) + { + throw new FormatException($"Failed to add HTTP header '{kv.Key}'."); + } + } + + #endregion constructor + + #region IDisposable + + public void Dispose() + { + if (_disposed) + return; + + _httpClient.Dispose(); + _disposed = true; + } + + #endregion IDisposable + + #region public + + public async Task ExportAsync(IReadOnlyList logs, CancellationToken token) + { + if (_disposed || logs.Count == 0 || token.IsCancellationRequested) + { + return; + } + + using RecyclableMemoryStream ms = _memoryManager.GetStream("HttpExport-Batch"); + + if (_ndJson) + { + LogBatchSerializer.WriteNdjson(ms, logs); + } + else + { + LogBatchSerializer.WriteJsonArray(ms, logs); + } + + ms.Position = 0; + + using StreamContent content = new StreamContent(ms); + content.Headers.ContentType = new MediaTypeHeaderValue( + _ndJson ? "application/x-ndjson" : "application/json"); + + using HttpResponseMessage response = await _httpClient + .PostAsync(_endpoint, content, token) + .ConfigureAwait(false); + + response.EnsureSuccessStatusCode(); + } + + #endregion public + } + + public static class LogBatchSerializer + { + private static readonly JsonWriterOptions WriterOptions = new JsonWriterOptions + { + Indented = false + }; + + public static void WriteNdjson(Stream target, IReadOnlyList logs) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(logs); + + if (logs.Count == 0) + { + return; + } + + ArrayBufferWriter buffer = new ArrayBufferWriter(4096); + using Utf8JsonWriter writer = new Utf8JsonWriter(buffer, WriterOptions); + + for (int i = 0; i < logs.Count; i++) + { + JsonSerializer.Serialize(writer, logs[i]); + writer.Flush(); + + target.Write(buffer.WrittenSpan); + target.WriteByte((byte)'\n'); + + buffer.Clear(); + + if (i < logs.Count - 1) + { + writer.Reset(buffer); + } + } + } + + public static void WriteJsonArray(Stream target, IReadOnlyList logs) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(logs); + + using Utf8JsonWriter writer = new Utf8JsonWriter(target, WriterOptions); + + writer.WriteStartArray(); + + for (int i = 0; i < logs.Count; i++) + { + JsonSerializer.Serialize(writer, logs[i]); + } + + writer.WriteEndArray(); + writer.Flush(); + } + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Sinks/IOutputSink.cs b/Apps/LogExporterApp/Sinks/IOutputSink.cs new file mode 100644 index 000000000..9e5f0c8f0 --- /dev/null +++ b/Apps/LogExporterApp/Sinks/IOutputSink.cs @@ -0,0 +1,39 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Sinks +{ + /// + /// Strategy interface to decide the sinks for exporting the logs. + /// ADR: Export operations must be cancellable so shutdown can abort long-running + /// network or disk operations. Without a cancellation token, HTTP, file, or syslog + /// sinks may block the DNS server shutdown indefinitely. All sinks must respect + /// the provided token to ensure bounded teardown. + /// + public interface IOutputSink: IDisposable + { + Task ExportAsync(IReadOnlyList logs, CancellationToken token); + } +} diff --git a/Apps/LogExporterApp/Sinks/NdjsonSerializer.cs b/Apps/LogExporterApp/Sinks/NdjsonSerializer.cs new file mode 100644 index 000000000..240804eb0 --- /dev/null +++ b/Apps/LogExporterApp/Sinks/NdjsonSerializer.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace LogExporter.Sinks +{ + /// + /// ADR: NDJSON serialization is used by all export strategies and must remain + /// consistent across sinks. Previously, each strategy copy/pasted its own + /// serialization loop, creating long-term maintenance and drift risks. + /// This helper centralizes NDJSON formatting so changes occur in one place, + /// ensuring consistency, reducing boilerplate, and eliminating subtle bugs. + /// + public static class NdjsonSerializer + { + private static readonly JsonWriterOptions WriterOptions = new JsonWriterOptions + { + Indented = false + }; + + public static void WriteBatch(Stream target, IReadOnlyList logs) + { + ArgumentNullException.ThrowIfNull(target); + ArgumentNullException.ThrowIfNull(logs); + + if (logs.Count == 0) + { + return; + } + + using Utf8JsonWriter writer = new Utf8JsonWriter(target, WriterOptions); + + for (int i = 0; i < logs.Count; i++) + { + JsonSerializer.Serialize(writer, logs[i]); + writer.Flush(); + + target.WriteByte((byte)'\n'); + + if (i < logs.Count - 1) + { + writer.Reset(target); + } + } + } + } +} diff --git a/Apps/LogExporterApp/Sinks/SinkDispatcher.cs b/Apps/LogExporterApp/Sinks/SinkDispatcher.cs new file mode 100644 index 000000000..850f6bf3c --- /dev/null +++ b/Apps/LogExporterApp/Sinks/SinkDispatcher.cs @@ -0,0 +1,111 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Sinks +{ + public sealed class SinkDispatcher : IDisposable + { + #region variables + + private readonly ConcurrentDictionary _sinks = + new ConcurrentDictionary(); + + private bool _disposed; + + #endregion + + #region IDisposable + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + + // ADR: Once the manager is disposed, all strategies must be disposed and + // removed. Leaving them in the dictionary creates a misleading state + // (“manager has strategies”) and allows accidental use-after-dispose. + // Clearing ensures the manager becomes inert and conveys finality. + foreach (KeyValuePair entry in _sinks) + entry.Value.Dispose(); + + _sinks.Clear(); + } + + #endregion + + #region public + + public void Add(IOutputSink sink) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + if (!_sinks.TryAdd(sink.GetType(), sink)) + throw new InvalidOperationException( + $"Strategy of type {sink.GetType().Name} already registered."); + } + + public void Remove(Type type) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + if (_sinks.TryRemove(type, out IOutputSink? existing)) + existing?.Dispose(); + } + + public bool Any() + { + if (_disposed) + return false; + + return !_sinks.IsEmpty; + } + + /// + /// Executes all configured export strategies for the current batch. + /// + /// ADR: ExportManager synchronously awaits each strategy's ExportAsync task. + /// This guarantees predictable backpressure and ensures no spillover work + /// continues after shutdown. Strategies are responsible for honoring + /// cancellation so shutdown stays bounded. + /// + public async Task DispatchAsync(IReadOnlyList logs, CancellationToken token) + { + if (_disposed || logs == null || logs.Count == 0 || _sinks.IsEmpty) + return; + + List tasks = new List(_sinks.Count); + + foreach (IOutputSink sink in _sinks.Values) + tasks.Add(sink.ExportAsync(logs, token)); + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + + #endregion + } +} diff --git a/Apps/LogExporterApp/Sinks/SyslogSink.cs b/Apps/LogExporterApp/Sinks/SyslogSink.cs new file mode 100644 index 000000000..f80567843 --- /dev/null +++ b/Apps/LogExporterApp/Sinks/SyslogSink.cs @@ -0,0 +1,309 @@ +/* +Technitium DNS Server +Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) +Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using Serilog; +using Serilog.Events; +using Serilog.Parsing; +using Serilog.Sinks.Syslog; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Sinks +{ + public sealed class SyslogSink : IOutputSink + { + #region variables + + const string _appName = "Technitium DNS Server"; + const string _sdId = "meta"; + const string DEFAULT_PROTOCOL = "udp"; + const int DEFAULT_PORT = 514; + + readonly Facility _facility = Facility.Local6; + + readonly Rfc5424Formatter _formatter; + readonly Serilog.Core.Logger _logger; + + bool _disposed; + + // Reuse the message template instead of parsing it per-log + const string TemplateText = "{questionsSummary}; RCODE: {rCode}; ANSWER: [{answersSummary}]"; + static readonly MessageTemplate Template = + new MessageTemplateParser().Parse(TemplateText); + + #endregion + + #region constructor + + public SyslogSink(string address, int? port, string? protocol) + { + port ??= DEFAULT_PORT; + protocol ??= DEFAULT_PROTOCOL; + + LoggerConfiguration conf = new LoggerConfiguration(); + + _logger = protocol.ToLowerInvariant() switch + { + "tls" => conf.WriteTo.TcpSyslog( + address, + port.Value, + _appName, + FramingType.OCTET_COUNTING, + SyslogFormat.RFC5424, + _facility, + useTls: true) + .Enrich.FromLogContext() + .CreateLogger(), + + "tcp" => conf.WriteTo.TcpSyslog( + address, + port.Value, + _appName, + FramingType.OCTET_COUNTING, + SyslogFormat.RFC5424, + _facility, + useTls: false) + .Enrich.FromLogContext() + .CreateLogger(), + + "udp" => conf.WriteTo.UdpSyslog( + address, + port.Value, + _appName, + SyslogFormat.RFC5424, + _facility) + .Enrich.FromLogContext() + .CreateLogger(), + + "local" => conf.WriteTo.LocalSyslog( + _appName, + _facility) + .Enrich.FromLogContext() + .CreateLogger(), + + _ => throw new NotSupportedException("SyslogSink protocol is not supported: " + protocol), + }; + + // Serilog's RFC5424 formatter used as before + _formatter = new Rfc5424Formatter( + facility: _facility, + applicationName: _appName, + templateFormatter: null, + messageIdPropertyName: _sdId, + sourceHost: Environment.MachineName, + severityMapping: null); + } + + #endregion + + #region IDisposable + + public void Dispose() + { + if (_disposed) + return; + + _logger.Dispose(); + _disposed = true; + } + + #endregion + + #region public + + public Task ExportAsync(IReadOnlyList logs, CancellationToken token) + { + // ADR: SyslogSink export previously used Task.Run with a synchronous loop, + // causing threadpool churn and preventing timely shutdown. We now execute + // sequentially on the caller's async context and check cancellation between + // log writes. Serilog remains synchronous, but cancellation ensures bounded + // shutdown latency. + + if (_disposed || logs.Count == 0 || token.IsCancellationRequested) + return Task.CompletedTask; + + foreach (LogEntry log in logs) + { + if (token.IsCancellationRequested) + break; + + string message = _formatter.FormatMessage(Convert(log)); + _logger.Information(message); + } + + return Task.CompletedTask; + } + + #endregion + + #region private + + private static LogEvent Convert(LogEntry log) + { + // Rough capacity: 9 base + 4 question + some answers + edns + // This avoids repeated List resizes + List properties = new List(16) + { + // Base fields (unchanged semantics) + new LogEventProperty( + "timestamp", + new ScalarValue(log.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))), + new LogEventProperty( + "clientIp", + new ScalarValue(log.ClientIp)), + new LogEventProperty( + "protocol", + new ScalarValue(log.Protocol.ToString())), + new LogEventProperty( + "responseType", + new ScalarValue(log.ResponseType.ToString())), + new LogEventProperty( + "responseRtt", + new ScalarValue(log.ResponseRtt?.ToString())), + new LogEventProperty( + "rCode", + new ScalarValue(log.ResponseCode.ToString())) + }; + + // Question + if (log.Question != null) + { + LogEntry.DnsQuestion question = log.Question; + + properties.Add(new LogEventProperty( + "qName", + new ScalarValue(question.QuestionName))); + + properties.Add(new LogEventProperty( + "qType", + new ScalarValue(question.QuestionType.ToString()))); + + properties.Add(new LogEventProperty( + "qClass", + new ScalarValue(question.QuestionClass.ToString()))); + + string questionSummary = + $"QNAME: {question.QuestionName}, " + + $"QTYPE: {question.QuestionType}, " + + $"QCLASS: {question.QuestionClass}"; + + properties.Add(new LogEventProperty( + "questionsSummary", + new ScalarValue(questionSummary))); + } + else + { + properties.Add(new LogEventProperty( + "questionsSummary", + new ScalarValue(string.Empty))); + } + + // Answers + if (log.Answers.Length > 0) + { + // Build answersSummary without LINQ + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < log.Answers.Length; i++) + { + LogEntry.DnsResourceRecord answer = log.Answers[i]; + + properties.Add(new LogEventProperty( + $"aName_{i}", + new ScalarValue(answer.Name))); + + properties.Add(new LogEventProperty( + $"aType_{i}", + new ScalarValue(answer.RecordType.ToString()))); + + properties.Add(new LogEventProperty( + $"aClass_{i}", + new ScalarValue(answer.RecordClass.ToString()))); + + properties.Add(new LogEventProperty( + $"aTtl_{i}", + new ScalarValue(answer.RecordTtl.ToString()))); + + properties.Add(new LogEventProperty( + $"aRData_{i}", + new ScalarValue(answer.RecordData))); + + properties.Add(new LogEventProperty( + $"aDnssecStatus_{i}", + new ScalarValue(answer.DnssecStatus.ToString()))); + + if (i > 0) + sb.Append(", "); + sb.Append(answer.RecordData); + } + + properties.Add(new LogEventProperty( + "answersSummary", + new ScalarValue(sb.ToString()))); + } + else + { + properties.Add(new LogEventProperty( + "answersSummary", + new ScalarValue(string.Empty))); + } + + // EDNS + if (log.EDNS.Length > 0) + { + for (int i = 0; i < log.EDNS.Length; i++) + { + LogEntry.EDNSLog ednsLog = log.EDNS[i]; + + properties.Add(new LogEventProperty( + $"ednsErrType_{i}", + new ScalarValue(ednsLog.ErrType))); + + properties.Add(new LogEventProperty( + $"ednsMessage_{i}", + new ScalarValue(ednsLog.Message))); + } + } + + // Meta + if (log.Meta.Count > 0) + { + foreach (var enrichment in log.Meta) + { + var k = enrichment.Key; + var v = enrichment.Value; + properties.Add(new LogEventProperty(k, new ScalarValue(v))); + } + } + + // Reuse the static MessageTemplate 'Template' + return new LogEvent( + timestamp: log.Timestamp, + level: LogEventLevel.Information, + exception: null, + messageTemplate: Template, + properties: properties); + } + + #endregion + } +} diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs deleted file mode 100644 index 867f9a779..000000000 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ /dev/null @@ -1,83 +0,0 @@ -/* -Technitium DNS Server -Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) -Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace LogExporter.Strategy -{ - public sealed class ExportManager : IDisposable - { - #region variables - - readonly ConcurrentDictionary _exportStrategies = new ConcurrentDictionary(); - - #endregion - - #region IDisposable - - public void Dispose() - { - foreach (KeyValuePair exportStrategy in _exportStrategies) - exportStrategy.Value.Dispose(); - } - - #endregion - - #region public - - public void AddStrategy(IExportStrategy strategy) - { - if (!_exportStrategies.TryAdd(strategy.GetType(), strategy)) - throw new InvalidOperationException(); - } - - public void RemoveStrategy(Type type) - { - if (_exportStrategies.TryRemove(type, out IExportStrategy? existing)) - existing?.Dispose(); - } - - public bool HasStrategy() - { - return !_exportStrategies.IsEmpty; - } - - public async Task ImplementStrategyAsync(IReadOnlyList logs) - { - List tasks = new List(_exportStrategies.Count); - - foreach (KeyValuePair strategy in _exportStrategies) - { - tasks.Add(Task.Factory.StartNew(delegate (object? state) - { - return strategy.Value.ExportAsync(logs); - }, null, CancellationToken.None, TaskCreationOptions.DenyChildAttach, TaskScheduler.Current)); - } - - await Task.WhenAll(tasks); - } - - #endregion - } -} diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs deleted file mode 100644 index e0797d068..000000000 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ /dev/null @@ -1,72 +0,0 @@ -/* -Technitium DNS Server -Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) -Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - -using Serilog; -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace LogExporter.Strategy -{ - public sealed class FileExportStrategy : IExportStrategy - { - #region variables - - readonly Serilog.Core.Logger _sender; - - bool _disposed; - - #endregion - - #region constructor - - public FileExportStrategy(string filePath) - { - _sender = new LoggerConfiguration().WriteTo.File(filePath, outputTemplate: "{Message:lj}{NewLine}{Exception}").CreateLogger(); - } - - #endregion - - #region IDisposable - - public void Dispose() - { - if (!_disposed) - { - _sender.Dispose(); - - _disposed = true; - } - } - - #endregion - - #region public - - public Task ExportAsync(IReadOnlyList logs) - { - foreach (LogEntry logEntry in logs) - _sender.Information(logEntry.ToString()); - - return Task.CompletedTask; - } - - #endregion - } -} diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs deleted file mode 100644 index 5318e0ba8..000000000 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ /dev/null @@ -1,121 +0,0 @@ -/* -Technitium DNS Server -Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) -Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - -using Microsoft.Extensions.Configuration; -using Serilog; -using Serilog.Sinks.Http; -using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace LogExporter.Strategy -{ - public sealed class HttpExportStrategy : IExportStrategy - { - #region variables - - readonly Serilog.Core.Logger _sender; - - bool _disposed; - - #endregion - - #region constructor - - public HttpExportStrategy(string endpoint, Dictionary? headers = null) - { - IConfigurationRoot? configuration = null; - if (headers != null) - { - configuration = new ConfigurationBuilder() - .AddInMemoryCollection(headers) - .Build(); - } - - _sender = new LoggerConfiguration().WriteTo.Http(endpoint, null, httpClient: new CustomHttpClient(), configuration: configuration).Enrich.FromLogContext().CreateLogger(); - } - - #endregion - - #region IDisposable - - public void Dispose() - { - if (!_disposed) - { - _sender.Dispose(); - - _disposed = true; - } - } - - #endregion - - #region public - - public Task ExportAsync(IReadOnlyList logs) - { - foreach (LogEntry logEntry in logs) - _sender.Information(logEntry.ToString()); - - return Task.CompletedTask; - } - - #endregion - - public class CustomHttpClient : IHttpClient - { - readonly HttpClient _httpClient; - - public CustomHttpClient() - { - _httpClient = new HttpClient(); - } - - public void Configure(IConfiguration configuration) - { - foreach (IConfigurationSection pair in configuration.GetChildren()) - { - if (!_httpClient.DefaultRequestHeaders.TryAddWithoutValidation(pair.Key, pair.Value)) - throw new FormatException($"Failed to add header '{pair.Key}'."); - } - } - - public void Dispose() - { - _httpClient?.Dispose(); - GC.SuppressFinalize(this); - } - - public async Task PostAsync(string requestUri, Stream contentStream, CancellationToken cancellationToken) - { - StreamContent content = new StreamContent(contentStream); - content.Headers.Add("Content-Type", "application/json"); - - return await _httpClient - .PostAsync(requestUri, content, cancellationToken) - .ConfigureAwait(false); - } - } - } -} diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs deleted file mode 100644 index 48b4e6401..000000000 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ /dev/null @@ -1,184 +0,0 @@ -/* -Technitium DNS Server -Copyright (C) 2025 Shreyas Zare (shreyas@technitium.com) -Copyright (C) 2025 Zafer Balkan (zafer@zaferbalkan.com) - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - -*/ - -using Serilog; -using Serilog.Events; -using Serilog.Parsing; -using Serilog.Sinks.Syslog; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace LogExporter.Strategy -{ - public sealed class SyslogExportStrategy : IExportStrategy - { - #region variables - - const string _appName = "Technitium DNS Server"; - const string _sdId = "meta"; - const string DEFAUL_PROTOCOL = "udp"; - const int DEFAULT_PORT = 514; - - readonly Facility _facility = Facility.Local6; - - readonly Rfc5424Formatter _formatter; - readonly Serilog.Core.Logger _sender; - - bool _disposed; - - #endregion - - #region constructor - - public SyslogExportStrategy(string address, int? port, string? protocol) - { - port ??= DEFAULT_PORT; - protocol ??= DEFAUL_PROTOCOL; - - LoggerConfiguration conf = new LoggerConfiguration(); - - _sender = protocol.ToLowerInvariant() switch - { - "tls" => conf.WriteTo.TcpSyslog(address, port.Value, _appName, FramingType.OCTET_COUNTING, SyslogFormat.RFC5424, _facility, useTls: true).Enrich.FromLogContext().CreateLogger(), - "tcp" => conf.WriteTo.TcpSyslog(address, port.Value, _appName, FramingType.OCTET_COUNTING, SyslogFormat.RFC5424, _facility, useTls: false).Enrich.FromLogContext().CreateLogger(), - "udp" => conf.WriteTo.UdpSyslog(address, port.Value, _appName, SyslogFormat.RFC5424, _facility).Enrich.FromLogContext().CreateLogger(), - "local" => conf.WriteTo.LocalSyslog(_appName, _facility).Enrich.FromLogContext().CreateLogger(), - _ => throw new NotSupportedException("Syslog protocol is not supported: " + protocol), - }; - - _formatter = new Rfc5424Formatter(_facility, _appName, null, _sdId, Environment.MachineName); - } - - #endregion - - #region IDisposable - - public void Dispose() - { - if (!_disposed) - { - _sender.Dispose(); - - _disposed = true; - } - } - - #endregion - - #region public - - public Task ExportAsync(IReadOnlyList logs) - { - foreach (LogEntry log in logs) - _sender.Information(_formatter.FormatMessage((LogEvent?)Convert(log))); - - return Task.CompletedTask; - } - - #endregion - - #region private - - private static LogEvent Convert(LogEntry log) - { - // Initialize properties with base log details - List properties = new List - { - new LogEventProperty("timestamp", new ScalarValue(log.Timestamp.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))), - new LogEventProperty("clientIp", new ScalarValue(log.ClientIp)), - new LogEventProperty("protocol", new ScalarValue(log.Protocol.ToString())), - new LogEventProperty("responseType", new ScalarValue(log.ResponseType.ToString())), - new LogEventProperty("responseRtt", new ScalarValue(log.ResponseRtt?.ToString())), - new LogEventProperty("rCode", new ScalarValue(log.ResponseCode.ToString())) - }; - - // Add each question as properties - if (log.Question != null) - { - LogEntry.DnsQuestion question = log.Question; - properties.Add(new LogEventProperty("qName", new ScalarValue(question.QuestionName))); - properties.Add(new LogEventProperty("qType", new ScalarValue(question.QuestionType.ToString()))); - properties.Add(new LogEventProperty("qClass", new ScalarValue(question.QuestionClass.ToString()))); - - string questionSummary = $"QNAME: {question.QuestionName}, QTYPE: {question.QuestionType}, QCLASS: {question.QuestionClass}"; - properties.Add(new LogEventProperty("questionsSummary", new ScalarValue(questionSummary))); - } - else - { - properties.Add(new LogEventProperty("questionsSummary", new ScalarValue(string.Empty))); - } - - // Add each answer as properties - if (log.Answers.Count > 0) - { - for (int i = 0; i < log.Answers.Count; i++) - { - LogEntry.DnsResourceRecord answer = log.Answers[i]; - - properties.Add(new LogEventProperty($"aName_{i}", new ScalarValue(answer.Name))); - properties.Add(new LogEventProperty($"aType_{i}", new ScalarValue(answer.RecordType.ToString()))); - properties.Add(new LogEventProperty($"aClass_{i}", new ScalarValue(answer.RecordClass.ToString()))); - properties.Add(new LogEventProperty($"aTtl_{i}", new ScalarValue(answer.RecordTtl.ToString()))); - properties.Add(new LogEventProperty($"aRData_{i}", new ScalarValue(answer.RecordData))); - properties.Add(new LogEventProperty($"aDnssecStatus_{i}", new ScalarValue(answer.DnssecStatus.ToString()))); - } - - // Generate answers summary - string answerSummary = string.Join(", ", log.Answers.Select(a => a.RecordData)); - properties.Add(new LogEventProperty("answersSummary", new ScalarValue(answerSummary))); - } - else - { - properties.Add(new LogEventProperty("answersSummary", new ScalarValue(string.Empty))); - } - - // Add EDNS logs - if (log.EDNS.Count > 0) - { - for (int i = 0; i < log.EDNS.Count; i++) - { - var ednsLog = log.EDNS[i]; - properties.Add(new LogEventProperty($"ednsErrType_{i}", new ScalarValue(ednsLog.ErrType))); - properties.Add(new LogEventProperty($"ednsMessage_{i}", new ScalarValue(ednsLog.Message))); - - } - } - - // Define the message template to match the original summary format - const string templateText = "{questionsSummary}; RCODE: {rCode}; ANSWER: [{answersSummary}]"; - - // Parse the template - MessageTemplate template = new MessageTemplateParser().Parse(templateText); - - // Create the LogEvent and return it - return new LogEvent( - timestamp: log.Timestamp, - level: LogEventLevel.Information, - exception: null, - messageTemplate: template, - properties: properties - ); - } - - #endregion - } -} diff --git a/Apps/LogExporterApp/dnsApp.config b/Apps/LogExporterApp/dnsApp.config index ce07081c2..8626d642e 100644 --- a/Apps/LogExporterApp/dnsApp.config +++ b/Apps/LogExporterApp/dnsApp.config @@ -1,21 +1,36 @@ { - "maxQueueSize": 1000000, - "ebableEdnsLogging": false, - "file": { - "path": "./dns_logs.json", - "enabled": false - }, - "http": { - "endpoint": "http://localhost:5000/logs", - "headers": { - "Authorization": "Bearer abc123" - }, - "enabled": false - }, - "syslog": { - "address": "127.0.0.1", - "port": 514, - "protocol": "UDP", - "enabled": false - } + "sinks":{ + "maxQueueSize": 1000000, + "enableEdnsLogging": false, + "console": { + "enabled": true + }, + "file": { + "path": "./dns_logs.json", + "enabled": false + }, + "http": { + "endpoint": "http://localhost:5000/logs", + "headers": { + "Authorization": "Bearer abc123" + }, + "enabled": false, + "ndjson": true + }, + "syslog": { + "address": "127.0.0.1", + "port": 514, + "protocol": "UDP", + "enabled": false + } + }, + "pipeline": { + "normalize": { + "enabled": true + }, + "tagging":{ + "enabled": false, + "tags": [ "tenant:alpha" ] + } + } } \ No newline at end of file