From 5980aab24eb501259bb1a879eff66eacf98e59b4 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Mon, 23 Sep 2024 22:41:23 +0300 Subject: [PATCH 01/17] Initial version of the Log Exporter app --- Apps/LogExporterApp/App.cs | 216 ++++++++++++++++++ Apps/LogExporterApp/BufferManagementConfig.cs | 70 ++++++ Apps/LogExporterApp/LogEntry.cs | 128 +++++++++++ Apps/LogExporterApp/LogExporterApp.csproj | 49 ++++ Apps/LogExporterApp/Strategy/ExportManager.cs | 53 +++++ .../Strategy/FileExportStrategy.cs | 84 +++++++ .../Strategy/HttpExportStrategy.cs | 91 ++++++++ .../Strategy/IExportStrategy.cs | 34 +++ .../Strategy/SyslogExportStrategy.cs | 168 ++++++++++++++ Apps/LogExporterApp/dnsApp.config | 26 +++ DnsServer.sln | 12 + 11 files changed, 931 insertions(+) create mode 100644 Apps/LogExporterApp/App.cs create mode 100644 Apps/LogExporterApp/BufferManagementConfig.cs create mode 100644 Apps/LogExporterApp/LogEntry.cs create mode 100644 Apps/LogExporterApp/LogExporterApp.csproj create mode 100644 Apps/LogExporterApp/Strategy/ExportManager.cs create mode 100644 Apps/LogExporterApp/Strategy/FileExportStrategy.cs create mode 100644 Apps/LogExporterApp/Strategy/HttpExportStrategy.cs create mode 100644 Apps/LogExporterApp/Strategy/IExportStrategy.cs create mode 100644 Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs create mode 100644 Apps/LogExporterApp/dnsApp.config diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs new file mode 100644 index 00000000..b0956388 --- /dev/null +++ b/Apps/LogExporterApp/App.cs @@ -0,0 +1,216 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using DnsServerCore.ApplicationCommon; +using LogExporter.Strategy; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using TechnitiumLibrary; +using TechnitiumLibrary.Net.Dns; +using TechnitiumLibrary.Net.Dns.ResourceRecords; + +namespace LogExporter +{ + public sealed class App : IDnsApplication, IDnsQueryLogger + { + #region variables + + private const int BULK_INSERT_COUNT = 1000; + + private const int DEFAULT_QUEUE_CAPACITY = 1000; + + private const int QUEUE_TIMER_INTERVAL = 10000; + + private readonly ExportManager _exportManager = new ExportManager(); + + private BlockingCollection _logBuffer; + + private readonly object _queueTimerLock = new object(); + + private BufferManagementConfig _config; + + private IDnsServer _dnsServer; + + private Timer _queueTimer; + + private bool disposedValue; + + private readonly IReadOnlyList _emptyList = []; + + #endregion variables + + #region constructor + + public App() + { + } + + #endregion constructor + + #region IDisposable + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + lock (_queueTimerLock) + { + _queueTimer?.Dispose(); + } + + ExportLogsAsync().Sync(); //flush any pending logs + + _logBuffer.Dispose(); + } + + disposedValue = true; + } + } + + public async Task InitializeAsync(IDnsServer dnsServer, string config) + { + _dnsServer = dnsServer; + _config = BufferManagementConfig.Deserialize(config); + if(_config == null) + { + throw new DnsClientException("Invalid application configuration."); + } + + if (_config.MaxLogEntries != null) + { + _logBuffer = new BlockingCollection(_config.MaxLogEntries.Value); + } + else + { + _logBuffer = new BlockingCollection(DEFAULT_QUEUE_CAPACITY); + } + + RegisterExportTargets(); + + lock (_queueTimerLock) + { + _queueTimer = new Timer(async (object _) => + { + try + { + await ExportLogsAsync(); + } + catch (Exception ex) + { + _dnsServer.WriteLog(ex); + } + }, null, QUEUE_TIMER_INTERVAL, Timeout.Infinite); + } + + await Task.CompletedTask; + } + + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + _logBuffer.Add(new LogEntry(timestamp, remoteEP, protocol, request, response)); + + return Task.CompletedTask; + } + + public async Task QueryLogsAsync(long pageNumber, int entriesPerPage, bool descendingOrder, DateTime? start, DateTime? end, IPAddress clientIpAddress, DnsTransportProtocol? protocol, DnsServerResponseType? responseType, DnsResponseCode? rcode, string qname, DnsResourceRecordType? qtype, DnsClass? qclass) + { + return await Task.FromResult(new DnsLogPage(0, 0, 0, _emptyList)); + } + + #endregion public + + #region private + + private async Task ExportLogsAsync(CancellationToken cancellationToken = default) + { + try + { + var logs = new List(BULK_INSERT_COUNT); + + while (true) + { + while ((logs.Count < BULK_INSERT_COUNT) && _logBuffer.TryTake(out LogEntry? log)) + { + if (log != null) + logs.Add(log); + } + + if (logs.Count > 0) + { + await _exportManager.ImplementStrategyForAsync(logs, cancellationToken); + + logs.Clear(); + } + } + } + catch (Exception ex) + { + _dnsServer?.WriteLog(ex); + } + } + + private void RegisterExportTargets() + { + foreach (var target in _config.Targets) + { + if (target.Enabled) + { + switch (target.Type.ToLower()) + { + case "file": + _exportManager.RegisterStrategy("file", new FileExportStrategy(target.Path)); + break; + + case "http": + _exportManager.RegisterStrategy("http", new HttpExportStrategy(target.Endpoint, target.Method, target.Headers)); + break; + + case "syslog": + _exportManager.RegisterStrategy("syslog", new SyslogExportStrategy(target.Address, target.Port, target.Protocol)); + break; + } + } + } + } + + #endregion private + + #region properties + + public string Description + { + get { return "The app allows exporting logs to a third party sink using an internal buffer."; } + } + + #endregion properties + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/BufferManagementConfig.cs b/Apps/LogExporterApp/BufferManagementConfig.cs new file mode 100644 index 00000000..396f3c5d --- /dev/null +++ b/Apps/LogExporterApp/BufferManagementConfig.cs @@ -0,0 +1,70 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace LogExporter +{ + public class BufferManagementConfig + { + [JsonPropertyName("maxLogEntries")] + public int? MaxLogEntries { get; set; } + + [JsonPropertyName("targets")] + public List Targets { get; set; } + + // Load configuration from JSON + public static BufferManagementConfig? Deserialize(string json) + { + return JsonSerializer.Deserialize(json); + } + } + + public class Target + { + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } + + [JsonPropertyName("path")] + public string Path { get; set; } + + [JsonPropertyName("endpoint")] + public string Endpoint { get; set; } + + [JsonPropertyName("method")] + public string Method { get; set; } + + [JsonPropertyName("headers")] + public Dictionary Headers { get; set; } + + [JsonPropertyName("address")] + public string Address { get; set; } + + [JsonPropertyName("port")] + public int? Port { get; set; } + + [JsonPropertyName("protocol")] + public string Protocol { get; set; } + } +} diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs new file mode 100644 index 00000000..1cc081e1 --- /dev/null +++ b/Apps/LogExporterApp/LogEntry.cs @@ -0,0 +1,128 @@ +using System; +using System.Linq; +using System.Net; +using System.Text.Json.Serialization; +using System.Text.Json; +using TechnitiumLibrary.Net.Dns; +using TechnitiumLibrary.Net.Dns.ResourceRecords; +using System.Collections.Generic; + +namespace LogExporter +{ + public class LogEntry + { + public DateTime Timestamp { get; set; } + public string ClientIp { get; set; } + public int ClientPort { get; set; } + public bool DnssecOk { get; set; } + public DnsTransportProtocol Protocol { get; set; } + public DnsResponseCode ResponseCode { get; set; } + public List Questions { get; set; } + public List Answers { get; set; } + public object? RequestTag { get; set; } + public object? ResponseTag { get; set; } + + public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response) + { + // Assign timestamp and ensure it's in UTC + Timestamp = timestamp.Kind == DateTimeKind.Utc ? timestamp : timestamp.ToUniversalTime(); + + // Extract client information + ClientIp = remoteEP.Address.ToString(); + ClientPort = remoteEP.Port; + DnssecOk = request.DnssecOk; + Protocol = protocol; + ResponseCode = response.RCODE; + + // Extract request information + Questions = new List(request.Question.Count); + if (request.Question?.Count > 0) + { + Questions.AddRange(request.Question.Select(questionRecord => new Question + { + QuestionName = questionRecord.Name, + QuestionType = questionRecord.Type, + QuestionClass = questionRecord.Class, + Size = questionRecord.UncompressedLength + })); + } + + // Convert answer section into a simple string summary (comma-separated for multiple answers) + Answers = new List(response.Answer.Count); + if (response.Answer?.Count > 0) + { + Answers.AddRange(response.Answer.Select(record => new Answer + { + RecordType = record.Type, + RecordData = record.RDATA.ToString(), + RecordClass = record.Class, + RecordTtl = record.TTL, + Size = record.UncompressedLength, + DnssecStatus = record.DnssecStatus + })); + } + + if (request.Tag != null) + { + RequestTag = request.Tag; + } + + if (response.Tag != null) + { + ResponseTag = response.Tag; + } + } + + public override string ToString() + { + return JsonSerializer.Serialize(this, DnsLogSerializerOptions.Default); + } + + public class Question + { + public string QuestionName { get; set; } + public DnsResourceRecordType? QuestionType { get; set; } + public DnsClass? QuestionClass { get; set; } + public int Size { get; set; } + } + + public class Answer + { + public DnsResourceRecordType RecordType { get; set; } + public string RecordData { get; set; } + public DnsClass RecordClass { get; set; } + public uint RecordTtl { get; set; } + public int Size { get; set; } + public DnssecStatus DnssecStatus { get; set; } + } + } + + // Custom DateTime converter to handle UTC serialization in ISO 8601 format + public class JsonDateTimeConverter : JsonConverter + { + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dts = reader.GetString(); + return dts == null ? DateTime.MinValue : DateTime.Parse(dts); + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ")); + } + } + + // Setup reusable options with a single instance + public static class DnsLogSerializerOptions + { + public static readonly JsonSerializerOptions Default = new JsonSerializerOptions + { + WriteIndented = false, // Newline delimited logs should not be multiline + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // Convert properties to camelCase + Converters = { new JsonStringEnumConverter(), new JsonDateTimeConverter() }, // Handle enums and DateTime conversion + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, // For safe encoding + NumberHandling = JsonNumberHandling.Strict, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull // Ignore null values + }; + } +} diff --git a/Apps/LogExporterApp/LogExporterApp.csproj b/Apps/LogExporterApp/LogExporterApp.csproj new file mode 100644 index 00000000..6c6107d2 --- /dev/null +++ b/Apps/LogExporterApp/LogExporterApp.csproj @@ -0,0 +1,49 @@ + + + + net8.0 + false + true + 1.0 + false + Technitium + Technitium DNS Server + Zafer Balkan + LogExporterApp + LogExporter + https://technitium.com/dns/ + https://github.com/TechnitiumSoftware/DnsServer + The app allows exporting logs to a third party sink using an internal buffer. + false + Library + enable + + + + + + + + + false + + + + + + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.Net.dll + false + + + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.dll + false + + + + + + PreserveNewest + + + + diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs new file mode 100644 index 00000000..abed40bc --- /dev/null +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -0,0 +1,53 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Strategy +{ + public class ExportManager + { + private readonly Dictionary _exportStrategies; + + public ExportManager() + { + _exportStrategies = new Dictionary(); + } + + public void RegisterStrategy(string key, IExportStrategy strategy) + { + _exportStrategies[key.ToLower()] = strategy; + } + + public IExportStrategy? GetStrategy(string key) + { + return _exportStrategies.ContainsKey(key.ToLower()) ? _exportStrategies[key.ToLower()] : null; + } + + public async Task ImplementStrategyForAsync(List logs, CancellationToken cancellationToken = default) + { + foreach (var strategy in _exportStrategies.Values) + { + await strategy.ExportLogsAsync(logs, cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs new file mode 100644 index 00000000..3f10cffa --- /dev/null +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -0,0 +1,84 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Strategy +{ + public class FileExportStrategy : IExportStrategy + { + private readonly string _filePath; + private static readonly SemaphoreSlim _fileSemaphore = new SemaphoreSlim(1, 1); + private bool disposedValue; + + public FileExportStrategy(string filePath) + { + _filePath = filePath; + } + + public async Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) + { + var jsonLogs = new StringBuilder(logs.Count); + foreach (var log in logs) + { + jsonLogs.AppendLine(log.ToString()); + } + + // Wait to enter the semaphore + await _fileSemaphore.WaitAsync(cancellationToken); + try + { + // Use a FileStream with exclusive access + using var fileStream = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(fileStream); + await writer.WriteAsync(jsonLogs.ToString()); + } + finally + { + // Release the semaphore + _fileSemaphore.Release(); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + _fileSemaphore.Release(); + _fileSemaphore.Dispose(); + } + + disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + System.GC.SuppressFinalize(this); + } + } +} diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs new file mode 100644 index 00000000..ce0157bb --- /dev/null +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -0,0 +1,91 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Strategy +{ + public class HttpExportStrategy : IExportStrategy + { + private readonly string _endpoint; + private readonly string _method; + private readonly Dictionary _headers; + private readonly HttpClient _httpClient; + private bool disposedValue; + + public HttpExportStrategy(string endpoint, string method, Dictionary headers) + { + _endpoint = endpoint; + _method = method; + _headers = headers; + _httpClient = new HttpClient(); + } + + public async Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) + { + var jsonLogs = new StringBuilder(logs.Count); + foreach (var log in logs) + { + jsonLogs.AppendLine(log.ToString()); + } + var request = new HttpRequestMessage + { + RequestUri = new Uri(_endpoint), + Method = new HttpMethod(_method), + Content = new StringContent(jsonLogs.ToString(), Encoding.UTF8, "application/json") + }; + + foreach (var header in _headers) + { + request.Headers.Add(header.Key, header.Value); + } + + var response = await _httpClient.SendAsync(request, cancellationToken); + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to export logs to {_endpoint}: {response.StatusCode}"); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + _httpClient.Dispose(); + } + + disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/Apps/LogExporterApp/Strategy/IExportStrategy.cs b/Apps/LogExporterApp/Strategy/IExportStrategy.cs new file mode 100644 index 00000000..c8aae94d --- /dev/null +++ b/Apps/LogExporterApp/Strategy/IExportStrategy.cs @@ -0,0 +1,34 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Strategy +{ + /// + /// Strategu interface to decide the sinks for exporting the logs. + /// + public interface IExportStrategy: IDisposable + { + Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default); + } +} diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs new file mode 100644 index 00000000..ab6c7624 --- /dev/null +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -0,0 +1,168 @@ +/* +Technitium DNS Server +Copyright (C) 2024 Shreyas Zare (shreyas@technitium.com) + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +*/ + +using SyslogNet.Client; +using SyslogNet.Client.Serialization; +using SyslogNet.Client.Transport; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace LogExporter.Strategy +{ + public class SyslogExportStrategy : IExportStrategy + { + private readonly string _processId; + private readonly string _host; + private const string _appName = "Technitium DNS Server"; + private const string _msgId = "dnslog"; + private const string _sdId = "dnsparams"; + + private readonly ISyslogMessageSender _sender; + private readonly ISyslogMessageSerializer _serializer; + private bool disposedValue; + + public SyslogExportStrategy(string address, int? port, string protocol = "udp") + { + port ??= 514; + _sender = protocol switch + { + "tls" => new SyslogEncryptedTcpSender(address, port.Value), + "tcp" => new SyslogTcpSender(address, port.Value), + "udp" => new SyslogUdpSender(address, port.Value), + "local" => new SyslogLocalSender(), + _ => throw new Exception("Invalid protocol specified"), + }; + + _serializer = new SyslogRfc5424MessageSerializer(); + _processId = Environment.ProcessId.ToString(); + _host = Environment.MachineName; + } + + public Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) + { + return Task.Run(() => + { + var messages = new List(logs.Select(Convert)); + _sender.Send(messages, _serializer); + + } + , cancellationToken); + } + + private SyslogMessage Convert(LogEntry log) + { + // Create the structured data with all key details from LogEntry + var elements = new StructuredDataElement(_sdId, new Dictionary + { + { "timestamp", log.Timestamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ") }, + { "clientIp", log.ClientIp }, + { "clientPort", log.ClientPort.ToString() }, + { "dnssecOk", log.DnssecOk.ToString() }, + { "protocol", log.Protocol.ToString() }, + { "rCode", log.ResponseCode.ToString() } + }); + + // Add each question to the structured data + if (log.Questions != null && log.Questions.Count > 0) + { + for (int i = 0; i < log.Questions.Count; i++) + { + var question = log.Questions[i]; + elements.Parameters.Add($"qName_{i}", question.QuestionName); + elements.Parameters.Add($"qType_{i}", question.QuestionType.HasValue ? question.QuestionType.Value.ToString() : "unknown"); + elements.Parameters.Add($"qClass_{i}", question.QuestionClass.HasValue ? question.QuestionClass.Value.ToString() : "unknown"); + elements.Parameters.Add($"qSize_{i}", question.Size.ToString()); + } + } + + // Add each answer to the structured data + if (log.Answers != null && log.Answers.Count > 0) + { + for (int i = 0; i < log.Answers.Count; i++) + { + var answer = log.Answers[i]; + elements.Parameters.Add($"aType_{i}", answer.RecordType.ToString()); + elements.Parameters.Add($"aData_{i}", answer.RecordData); + elements.Parameters.Add($"aClass_{i}", answer.RecordClass.ToString()); + elements.Parameters.Add($"aTtl_{i}", answer.RecordTtl.ToString()); + elements.Parameters.Add($"aSize_{i}", answer.Size.ToString()); + elements.Parameters.Add($"aDnssecStatus_{i}", answer.DnssecStatus.ToString()); + } + } + + // Include request and response tags if present + if (log.RequestTag != null) + { + elements.Parameters.Add("requestTag", log.RequestTag.ToString()); + } + + if (log.ResponseTag != null) + { + elements.Parameters.Add("responseTag", log.ResponseTag.ToString()); + } + + // Build a comprehensive message summary + string questionSummary = log.Questions?.Count > 0 + ? string.Join(", ", log.Questions.Select((q, index) => $"{q.QuestionName} (Type: {q.QuestionType}, Class: {q.QuestionClass}, Size: {q.Size})")) + : "No Questions"; + + string answerSummary = log.Answers?.Count > 0 + ? string.Join(", ", log.Answers.Select((a, index) => $"{a.RecordData} (Type: {a.RecordType}, Class: {a.RecordClass}, TTL: {a.RecordTtl}, Size: {a.Size}, DNSSEC: {a.DnssecStatus})")) + : "No Answers"; + + string messageSummary = $"{log.ClientIp}:{log.ClientPort} {log.Protocol} DNSSEC={log.DnssecOk} {questionSummary} {log.ResponseCode} {answerSummary}"; + + // Create and return the syslog message + return new SyslogMessage( + log.Timestamp, + Facility.UserLevelMessages, + Severity.Informational, + _host, + _appName, + _processId, + _msgId, + messageSummary, + elements + ); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + _sender.Dispose(); + } + + disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/Apps/LogExporterApp/dnsApp.config b/Apps/LogExporterApp/dnsApp.config new file mode 100644 index 00000000..0766f415 --- /dev/null +++ b/Apps/LogExporterApp/dnsApp.config @@ -0,0 +1,26 @@ +{ + "maxLogEntries": 10000, + "targets": [ + { + "type": "file", + "enabled": true, + "path": "/var/log/dns_logs.json" + }, + { + "type": "http", + "enabled": false, + "endpoint": "http://example.com/logs", + "method": "POST", + "headers": { + "Authorization": "Bearer abc123" + } + }, + { + "type": "syslog", + "enabled": false, + "address": "127.0.0.1", + "port": 514, + "protocol": "udp" + } + ] +} diff --git a/DnsServer.sln b/DnsServer.sln index 06764378..05f4513d 100644 --- a/DnsServer.sln +++ b/DnsServer.sln @@ -31,6 +31,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DropRequestsApp", "Apps\Dro EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QueryLogsSqliteApp", "Apps\QueryLogsSqliteApp\QueryLogsSqliteApp.csproj", "{186DEF23-863E-4954-BE16-5E5FCA75ECA2}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogExporterApp", "Apps\LogExporterApp\LogExporterApp.csproj", "{6F9BCCA9-6422-484B-A065-EF8AF9DA74B5}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AdvancedBlockingApp", "Apps\AdvancedBlockingApp\AdvancedBlockingApp.csproj", "{A4C31093-CA65-42D4-928A-11907076C0DE}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NxDomainApp", "Apps\NxDomainApp\NxDomainApp.csproj", "{BB0010FC-20E9-4397-BF9B-C9955D9AD339}" @@ -61,6 +63,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DnsRebindingProtectionApp", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FilterAaaaApp", "Apps\FilterAaaaApp\FilterAaaaApp.csproj", "{0A9B7F39-80DA-4084-AD47-8707576927ED}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{3746EF13-91C5-4858-9DC2-D3C2504BD135}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -119,6 +126,10 @@ Global {186DEF23-863E-4954-BE16-5E5FCA75ECA2}.Debug|Any CPU.Build.0 = Debug|Any CPU {186DEF23-863E-4954-BE16-5E5FCA75ECA2}.Release|Any CPU.ActiveCfg = Release|Any CPU {186DEF23-863E-4954-BE16-5E5FCA75ECA2}.Release|Any CPU.Build.0 = Release|Any CPU + {6F9BCCA9-6422-484B-A065-EF8AF9DA74B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F9BCCA9-6422-484B-A065-EF8AF9DA74B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F9BCCA9-6422-484B-A065-EF8AF9DA74B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F9BCCA9-6422-484B-A065-EF8AF9DA74B5}.Release|Any CPU.Build.0 = Release|Any CPU {A4C31093-CA65-42D4-928A-11907076C0DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A4C31093-CA65-42D4-928A-11907076C0DE}.Debug|Any CPU.Build.0 = Debug|Any CPU {A4C31093-CA65-42D4-928A-11907076C0DE}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -192,6 +203,7 @@ Global {099D27AF-3AEB-495A-A5D0-46DA59CC9213} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {738079D1-FA5A-40CD-8A27-D831919EE209} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {186DEF23-863E-4954-BE16-5E5FCA75ECA2} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} + {6F9BCCA9-6422-484B-A065-EF8AF9DA74B5} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {A4C31093-CA65-42D4-928A-11907076C0DE} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {BB0010FC-20E9-4397-BF9B-C9955D9AD339} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} {45C6F9AD-57D6-4D6D-9498-10B5C828E47E} = {938BF8EF-74B9-4FE0-B46F-11EBB7A4B3D2} From 7a934d9eb854625729390650f0430e264abae6fd Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Sun, 29 Sep 2024 11:42:21 +0300 Subject: [PATCH 02/17] Regions --- Apps/LogExporterApp/BufferManagementConfig.cs | 4 +- Apps/LogExporterApp/Strategy/ExportManager.cs | 22 +++-- .../Strategy/FileExportStrategy.cs | 35 +++++--- .../Strategy/HttpExportStrategy.cs | 47 ++++++++--- .../Strategy/SyslogExportStrategy.cs | 81 +++++++++++++------ 5 files changed, 136 insertions(+), 53 deletions(-) diff --git a/Apps/LogExporterApp/BufferManagementConfig.cs b/Apps/LogExporterApp/BufferManagementConfig.cs index 396f3c5d..836c7958 100644 --- a/Apps/LogExporterApp/BufferManagementConfig.cs +++ b/Apps/LogExporterApp/BufferManagementConfig.cs @@ -56,7 +56,7 @@ namespace LogExporter public string Method { get; set; } [JsonPropertyName("headers")] - public Dictionary Headers { get; set; } + public Dictionary? Headers { get; set; } [JsonPropertyName("address")] public string Address { get; set; } @@ -65,6 +65,6 @@ namespace LogExporter public int? Port { get; set; } [JsonPropertyName("protocol")] - public string Protocol { get; set; } + public string? Protocol { get; set; } } } diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs index abed40bc..501996b5 100644 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -25,17 +25,22 @@ namespace LogExporter.Strategy { public class ExportManager { + #region variables + private readonly Dictionary _exportStrategies; + #endregion variables + + #region constructor + public ExportManager() { _exportStrategies = new Dictionary(); } - public void RegisterStrategy(string key, IExportStrategy strategy) - { - _exportStrategies[key.ToLower()] = strategy; - } + #endregion constructor + + #region public public IExportStrategy? GetStrategy(string key) { @@ -49,5 +54,12 @@ namespace LogExporter.Strategy await strategy.ExportLogsAsync(logs, cancellationToken).ConfigureAwait(false); } } + + public void RegisterStrategy(string key, IExportStrategy strategy) + { + _exportStrategies[key.ToLower()] = strategy; + } + + #endregion public } -} +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 3f10cffa..8b4d6d6d 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -27,15 +27,27 @@ namespace LogExporter.Strategy { public class FileExportStrategy : IExportStrategy { - private readonly string _filePath; + #region variables + private static readonly SemaphoreSlim _fileSemaphore = new SemaphoreSlim(1, 1); + + private readonly string _filePath; + private bool disposedValue; + #endregion variables + + #region constructor + public FileExportStrategy(string filePath) { _filePath = filePath; } + #endregion constructor + + #region public + public async Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) { var jsonLogs = new StringBuilder(logs.Count); @@ -60,6 +72,17 @@ namespace LogExporter.Strategy } } + #endregion public + + #region IDisposable + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + System.GC.SuppressFinalize(this); + } + protected virtual void Dispose(bool disposing) { if (!disposedValue) @@ -73,12 +96,6 @@ namespace LogExporter.Strategy disposedValue = true; } } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - System.GC.SuppressFinalize(this); - } + #endregion IDisposable } -} +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index ce0157bb..a3d8d769 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -28,13 +28,23 @@ namespace LogExporter.Strategy { public class HttpExportStrategy : IExportStrategy { + #region variables + private readonly string _endpoint; - private readonly string _method; - private readonly Dictionary _headers; + + private readonly Dictionary? _headers; + private readonly HttpClient _httpClient; + + private readonly string _method; + private bool disposedValue; - public HttpExportStrategy(string endpoint, string method, Dictionary headers) + #endregion variables + + #region constructor + + public HttpExportStrategy(string endpoint, string method, Dictionary? headers) { _endpoint = endpoint; _method = method; @@ -42,6 +52,10 @@ namespace LogExporter.Strategy _httpClient = new HttpClient(); } + #endregion constructor + + #region public + public async Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) { var jsonLogs = new StringBuilder(logs.Count); @@ -56,9 +70,12 @@ namespace LogExporter.Strategy Content = new StringContent(jsonLogs.ToString(), Encoding.UTF8, "application/json") }; - foreach (var header in _headers) + if (_headers != null) { - request.Headers.Add(header.Key, header.Value); + foreach (var header in _headers) + { + request.Headers.Add(header.Key, header.Value); + } } var response = await _httpClient.SendAsync(request, cancellationToken); @@ -68,6 +85,17 @@ namespace LogExporter.Strategy } } + #endregion public + + #region IDisposable + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + protected virtual void Dispose(bool disposing) { if (!disposedValue) @@ -81,11 +109,6 @@ namespace LogExporter.Strategy } } - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } + #endregion IDisposable } -} +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index ab6c7624..94c25648 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -30,19 +30,37 @@ namespace LogExporter.Strategy { public class SyslogExportStrategy : IExportStrategy { - private readonly string _processId; - private readonly string _host; + #region variables + private const string _appName = "Technitium DNS Server"; + private const string _msgId = "dnslog"; + private const string _sdId = "dnsparams"; + private const string DEFAUL_PROTOCOL = "udp"; + + private const int DEFAULT_PORT = 514; + + private readonly string _host; + + private readonly string _processId; + private readonly ISyslogMessageSender _sender; + private readonly ISyslogMessageSerializer _serializer; + private bool disposedValue; - public SyslogExportStrategy(string address, int? port, string protocol = "udp") + #endregion variables + + #region constructor + + public SyslogExportStrategy(string address, int? port, string? protocol) { - port ??= 514; + port ??= DEFAULT_PORT; + protocol ??= DEFAUL_PROTOCOL; + _sender = protocol switch { "tls" => new SyslogEncryptedTcpSender(address, port.Value), @@ -57,17 +75,48 @@ namespace LogExporter.Strategy _host = Environment.MachineName; } + #endregion constructor + + #region public + public Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) { return Task.Run(() => { var messages = new List(logs.Select(Convert)); _sender.Send(messages, _serializer); - } , cancellationToken); } + #endregion public + + #region IDisposable + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + _sender.Dispose(); + } + + disposedValue = true; + } + } + + #endregion IDisposable + + #region private + private SyslogMessage Convert(LogEntry log) { // Create the structured data with all key details from LogEntry @@ -145,24 +194,6 @@ namespace LogExporter.Strategy ); } - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - _sender.Dispose(); - } - - disposedValue = true; - } - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } + #endregion private } -} +} \ No newline at end of file From 81b553761f3a108206d56886dd8046447df36158 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 1 Oct 2024 14:02:16 +0300 Subject: [PATCH 03/17] Simplified syslog message --- .../Strategy/SyslogExportStrategy.cs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index 94c25648..35f8b9ca 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -121,14 +121,14 @@ namespace LogExporter.Strategy { // Create the structured data with all key details from LogEntry var elements = new StructuredDataElement(_sdId, new Dictionary - { - { "timestamp", log.Timestamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ") }, - { "clientIp", log.ClientIp }, - { "clientPort", log.ClientPort.ToString() }, - { "dnssecOk", log.DnssecOk.ToString() }, - { "protocol", log.Protocol.ToString() }, - { "rCode", log.ResponseCode.ToString() } - }); + { + { "timestamp", log.Timestamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ") }, + { "clientIp", log.ClientIp }, + { "clientPort", log.ClientPort.ToString() }, + { "dnssecOk", log.DnssecOk.ToString() }, + { "protocol", log.Protocol.ToString() }, + { "rCode", log.ResponseCode.ToString() } + }); // Add each question to the structured data if (log.Questions != null && log.Questions.Count > 0) @@ -171,14 +171,18 @@ namespace LogExporter.Strategy // Build a comprehensive message summary string questionSummary = log.Questions?.Count > 0 - ? string.Join(", ", log.Questions.Select((q, index) => $"{q.QuestionName} (Type: {q.QuestionType}, Class: {q.QuestionClass}, Size: {q.Size})")) - : "No Questions"; + ? string.Join("; ", log.Questions.Select(q => + $"QNAME: {q.QuestionName}; QTYPE: {q.QuestionType?.ToString() ?? "unknown"}; QCLASS: {q.QuestionClass?.ToString() ?? "unknown"}")) + : "No Questions"; + // Build the answer summary in the desired format string answerSummary = log.Answers?.Count > 0 - ? string.Join(", ", log.Answers.Select((a, index) => $"{a.RecordData} (Type: {a.RecordType}, Class: {a.RecordClass}, TTL: {a.RecordTtl}, Size: {a.Size}, DNSSEC: {a.DnssecStatus})")) + ? string.Join(", ", log.Answers.Select(a => a.RecordData)) : "No Answers"; - string messageSummary = $"{log.ClientIp}:{log.ClientPort} {log.Protocol} DNSSEC={log.DnssecOk} {questionSummary} {log.ResponseCode} {answerSummary}"; + // Construct the message summary string to match the desired format + string messageSummary = $"{questionSummary}; RCODE: {log.ResponseCode}; ANSWER: [{answerSummary}]"; + // Create and return the syslog message return new SyslogMessage( From 952290dce15e399ceb665e36a01047f56d7e57ee Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Tue, 1 Oct 2024 14:20:56 +0300 Subject: [PATCH 04/17] Aligned targets and strategies --- Apps/LogExporterApp/App.cs | 46 +++++++++------- Apps/LogExporterApp/BufferManagementConfig.cs | 52 ++++++++++++------- Apps/LogExporterApp/Strategy/ExportManager.cs | 15 ++---- 3 files changed, 65 insertions(+), 48 deletions(-) diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index b0956388..864b46c3 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -96,11 +96,15 @@ namespace LogExporter } } - public async Task InitializeAsync(IDnsServer dnsServer, string config) + #endregion IDisposable + + #region public + + public Task InitializeAsync(IDnsServer dnsServer, string config) { _dnsServer = dnsServer; _config = BufferManagementConfig.Deserialize(config); - if(_config == null) + if (_config == null) { throw new DnsClientException("Invalid application configuration."); } @@ -131,7 +135,7 @@ namespace LogExporter }, null, QUEUE_TIMER_INTERVAL, Timeout.Infinite); } - await Task.CompletedTask; + return Task.CompletedTask; } public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) @@ -180,25 +184,29 @@ namespace LogExporter private void RegisterExportTargets() { - foreach (var target in _config.Targets) + + var fileTarget = _config.FileTarget; + if (fileTarget != null && fileTarget.Enabled) { - if (target.Enabled) - { - switch (target.Type.ToLower()) - { - case "file": - _exportManager.RegisterStrategy("file", new FileExportStrategy(target.Path)); - break; + var strategy = new FileExportStrategy(fileTarget.Path); + _exportManager.RegisterStrategy(strategy); - case "http": - _exportManager.RegisterStrategy("http", new HttpExportStrategy(target.Endpoint, target.Method, target.Headers)); - break; + } + + var httpTarget = _config.HttpTarget; + if (httpTarget != null && httpTarget.Enabled) + { + var strategy = new HttpExportStrategy(httpTarget.Endpoint, httpTarget.Method, httpTarget.Headers); + _exportManager.RegisterStrategy(strategy); + + } + + var syslogTarget = _config.SyslogTarget; + if (syslogTarget != null && syslogTarget.Enabled) + { + var strategy = new SyslogExportStrategy(syslogTarget.Address, syslogTarget.Port, syslogTarget.Protocol); + _exportManager.RegisterStrategy(strategy); - case "syslog": - _exportManager.RegisterStrategy("syslog", new SyslogExportStrategy(target.Address, target.Port, target.Protocol)); - break; - } - } } } diff --git a/Apps/LogExporterApp/BufferManagementConfig.cs b/Apps/LogExporterApp/BufferManagementConfig.cs index 836c7958..a75661d5 100644 --- a/Apps/LogExporterApp/BufferManagementConfig.cs +++ b/Apps/LogExporterApp/BufferManagementConfig.cs @@ -28,8 +28,14 @@ namespace LogExporter [JsonPropertyName("maxLogEntries")] public int? MaxLogEntries { get; set; } - [JsonPropertyName("targets")] - public List Targets { get; set; } + [JsonPropertyName("file")] + public FileTarget? FileTarget { get; set; } + + [JsonPropertyName("http")] + public HttpTarget? HttpTarget { get; set; } + + [JsonPropertyName("syslog")] + public SyslogTarget? SyslogTarget { get; set; } // Load configuration from JSON public static BufferManagementConfig? Deserialize(string json) @@ -37,27 +43,11 @@ namespace LogExporter return JsonSerializer.Deserialize(json); } } - - public class Target + public class SyslogTarget { - [JsonPropertyName("type")] - public string Type { get; set; } - [JsonPropertyName("enabled")] public bool Enabled { get; set; } - [JsonPropertyName("path")] - public string Path { get; set; } - - [JsonPropertyName("endpoint")] - public string Endpoint { get; set; } - - [JsonPropertyName("method")] - public string Method { get; set; } - - [JsonPropertyName("headers")] - public Dictionary? Headers { get; set; } - [JsonPropertyName("address")] public string Address { get; set; } @@ -67,4 +57,28 @@ namespace LogExporter [JsonPropertyName("protocol")] public string? Protocol { get; set; } } + + public class FileTarget + { + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } + + [JsonPropertyName("path")] + public string Path { get; set; } + } + + public class HttpTarget + { + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } + + [JsonPropertyName("endpoint")] + public string Endpoint { get; set; } + + [JsonPropertyName("method")] + public string Method { get; set; } + + [JsonPropertyName("headers")] + public Dictionary? Headers { get; set; } + } } diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs index 501996b5..eecc7999 100644 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -27,7 +27,7 @@ namespace LogExporter.Strategy { #region variables - private readonly Dictionary _exportStrategies; + private readonly List _exportStrategies; #endregion variables @@ -35,29 +35,24 @@ namespace LogExporter.Strategy public ExportManager() { - _exportStrategies = new Dictionary(); + _exportStrategies = new List(); } #endregion constructor #region public - public IExportStrategy? GetStrategy(string key) - { - return _exportStrategies.ContainsKey(key.ToLower()) ? _exportStrategies[key.ToLower()] : null; - } - public async Task ImplementStrategyForAsync(List logs, CancellationToken cancellationToken = default) { - foreach (var strategy in _exportStrategies.Values) + foreach (var strategy in _exportStrategies) { await strategy.ExportLogsAsync(logs, cancellationToken).ConfigureAwait(false); } } - public void RegisterStrategy(string key, IExportStrategy strategy) + public void RegisterStrategy(IExportStrategy strategy) { - _exportStrategies[key.ToLower()] = strategy; + _exportStrategies.Add(strategy); } #endregion public From 7425191a500e8095b0d6f889c4bbd2fc6bf69b03 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 2 Oct 2024 20:02:44 +0300 Subject: [PATCH 05/17] Simplified config --- Apps/LogExporterApp/App.cs | 37 +++++++-------- Apps/LogExporterApp/BufferManagementConfig.cs | 16 +++---- Apps/LogExporterApp/Strategy/ExportManager.cs | 19 +++++--- .../Strategy/SyslogExportStrategy.cs | 2 +- Apps/LogExporterApp/dnsApp.config | 45 +++++++++---------- 5 files changed, 56 insertions(+), 63 deletions(-) diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index 864b46c3..9f9188b3 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -104,6 +104,7 @@ namespace LogExporter { _dnsServer = dnsServer; _config = BufferManagementConfig.Deserialize(config); + if (_config == null) { throw new DnsClientException("Invalid application configuration."); @@ -160,7 +161,7 @@ namespace LogExporter { var logs = new List(BULK_INSERT_COUNT); - while (true) + while (!cancellationToken.IsCancellationRequested) { while ((logs.Count < BULK_INSERT_COUNT) && _logBuffer.TryTake(out LogEntry? log)) { @@ -184,30 +185,22 @@ namespace LogExporter private void RegisterExportTargets() { - - var fileTarget = _config.FileTarget; - if (fileTarget != null && fileTarget.Enabled) + // Helper function to register an export strategy if the target is enabled + void RegisterIfEnabled(TTarget target, Func strategyFactory) + where TTarget : TargetBase + where TStrategy : IExportStrategy { - var strategy = new FileExportStrategy(fileTarget.Path); - _exportManager.RegisterStrategy(strategy); - + if (target?.Enabled == true) + { + var strategy = strategyFactory(target); + _exportManager.AddOrReplaceStrategy(strategy); + } } - var httpTarget = _config.HttpTarget; - if (httpTarget != null && httpTarget.Enabled) - { - var strategy = new HttpExportStrategy(httpTarget.Endpoint, httpTarget.Method, httpTarget.Headers); - _exportManager.RegisterStrategy(strategy); - - } - - var syslogTarget = _config.SyslogTarget; - if (syslogTarget != null && syslogTarget.Enabled) - { - var strategy = new SyslogExportStrategy(syslogTarget.Address, syslogTarget.Port, syslogTarget.Protocol); - _exportManager.RegisterStrategy(strategy); - - } + // Register the different strategies using the helper + RegisterIfEnabled(_config.FileTarget, target => new FileExportStrategy(target.Path)); + RegisterIfEnabled(_config.HttpTarget, target => new HttpExportStrategy(target.Endpoint, target.Method, target.Headers)); + RegisterIfEnabled(_config.SyslogTarget, target => new SyslogExportStrategy(target.Address, target.Port, target.Protocol)); } #endregion private diff --git a/Apps/LogExporterApp/BufferManagementConfig.cs b/Apps/LogExporterApp/BufferManagementConfig.cs index a75661d5..4ef64430 100644 --- a/Apps/LogExporterApp/BufferManagementConfig.cs +++ b/Apps/LogExporterApp/BufferManagementConfig.cs @@ -43,11 +43,15 @@ namespace LogExporter return JsonSerializer.Deserialize(json); } } - public class SyslogTarget + + public class TargetBase { [JsonPropertyName("enabled")] public bool Enabled { get; set; } + } + public class SyslogTarget : TargetBase + { [JsonPropertyName("address")] public string Address { get; set; } @@ -58,20 +62,14 @@ namespace LogExporter public string? Protocol { get; set; } } - public class FileTarget + public class FileTarget : TargetBase { - [JsonPropertyName("enabled")] - public bool Enabled { get; set; } - [JsonPropertyName("path")] public string Path { get; set; } } - public class HttpTarget + public class HttpTarget : TargetBase { - [JsonPropertyName("enabled")] - public bool Enabled { get; set; } - [JsonPropertyName("endpoint")] public string Endpoint { get; set; } diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs index eecc7999..64aac98a 100644 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -17,6 +17,7 @@ along with this program. If not, see . */ +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -27,7 +28,7 @@ namespace LogExporter.Strategy { #region variables - private readonly List _exportStrategies; + private readonly Dictionary _exportStrategies; #endregion variables @@ -35,24 +36,30 @@ namespace LogExporter.Strategy public ExportManager() { - _exportStrategies = new List(); + _exportStrategies = new Dictionary(); } #endregion constructor #region public + public IExportStrategy? GetStrategy() where T : IExportStrategy + { + _exportStrategies.TryGetValue(typeof(T), out var strategy); + return strategy; + } + public async Task ImplementStrategyForAsync(List logs, CancellationToken cancellationToken = default) { - foreach (var strategy in _exportStrategies) + foreach (var strategy in _exportStrategies.Values) { - await strategy.ExportLogsAsync(logs, cancellationToken).ConfigureAwait(false); + await strategy.ExportLogsAsync(logs, cancellationToken); } } - public void RegisterStrategy(IExportStrategy strategy) + public void AddOrReplaceStrategy(IExportStrategy strategy) { - _exportStrategies.Add(strategy); + _exportStrategies[strategy.GetType()] = strategy; } #endregion public diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index 35f8b9ca..303ab2c1 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -61,7 +61,7 @@ namespace LogExporter.Strategy port ??= DEFAULT_PORT; protocol ??= DEFAUL_PROTOCOL; - _sender = protocol switch + _sender = protocol.ToLowerInvariant() switch { "tls" => new SyslogEncryptedTcpSender(address, port.Value), "tcp" => new SyslogTcpSender(address, port.Value), diff --git a/Apps/LogExporterApp/dnsApp.config b/Apps/LogExporterApp/dnsApp.config index 0766f415..beb75381 100644 --- a/Apps/LogExporterApp/dnsApp.config +++ b/Apps/LogExporterApp/dnsApp.config @@ -1,26 +1,21 @@ { - "maxLogEntries": 10000, - "targets": [ - { - "type": "file", - "enabled": true, - "path": "/var/log/dns_logs.json" - }, - { - "type": "http", - "enabled": false, - "endpoint": "http://example.com/logs", - "method": "POST", - "headers": { - "Authorization": "Bearer abc123" - } - }, - { - "type": "syslog", - "enabled": false, - "address": "127.0.0.1", - "port": 514, - "protocol": "udp" - } - ] -} + "maxLogEntries": 1000, + "file": { + "path": "/var/log/dns_logs.json", + "enabled": true + }, + "http": { + "endpoint": "http://example.com/logs", + "method": "POST", + "headers": { + "Authorization": "Bearer abc123" + }, + "enabled": true + }, + "syslog": { + "address": "127.0.0.1", + "port": 514, + "protocol": "UDP", + "enabled": true + } +} \ No newline at end of file From fb42498485e1db978bebf5db5afc179f201b5e05 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 3 Oct 2024 15:14:29 +0300 Subject: [PATCH 06/17] Performance improvements Signed-off-by: Zafer Balkan --- Apps/LogExporterApp/App.cs | 83 +++++------ Apps/LogExporterApp/BufferManagementConfig.cs | 16 ++- Apps/LogExporterApp/GrowableBuffer.cs | 132 ++++++++++++++++++ Apps/LogExporterApp/LogEntry.cs | 101 +++++++++----- Apps/LogExporterApp/Strategy/ExportManager.cs | 28 ++-- .../Strategy/FileExportStrategy.cs | 37 +++-- .../Strategy/HttpExportStrategy.cs | 13 +- .../Strategy/IExportStrategy.cs | 3 +- .../Strategy/SyslogExportStrategy.cs | 13 +- Apps/LogExporterApp/dnsApp.config | 6 +- 10 files changed, 309 insertions(+), 123 deletions(-) create mode 100644 Apps/LogExporterApp/GrowableBuffer.cs diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index 9f9188b3..5030de81 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -41,22 +41,20 @@ namespace LogExporter private const int QUEUE_TIMER_INTERVAL = 10000; + private readonly IReadOnlyList _emptyList = []; + private readonly ExportManager _exportManager = new ExportManager(); - private BlockingCollection _logBuffer; - - private readonly object _queueTimerLock = new object(); - - private BufferManagementConfig _config; + private BufferManagementConfig? _config; private IDnsServer _dnsServer; + private BlockingCollection _logBuffer; + private Timer _queueTimer; private bool disposedValue; - private readonly IReadOnlyList _emptyList = []; - #endregion variables #region constructor @@ -82,10 +80,7 @@ namespace LogExporter { if (disposing) { - lock (_queueTimerLock) - { - _queueTimer?.Dispose(); - } + _queueTimer?.Dispose(); ExportLogsAsync().Sync(); //flush any pending logs @@ -120,22 +115,10 @@ namespace LogExporter } RegisterExportTargets(); - - lock (_queueTimerLock) + if (_exportManager.HasStrategy()) { - _queueTimer = new Timer(async (object _) => - { - try - { - await ExportLogsAsync(); - } - catch (Exception ex) - { - _dnsServer.WriteLog(ex); - } - }, null, QUEUE_TIMER_INTERVAL, Timeout.Infinite); + _queueTimer = new Timer(HandleExportLogCallback, state: null, QUEUE_TIMER_INTERVAL, Timeout.Infinite); } - return Task.CompletedTask; } @@ -146,41 +129,51 @@ namespace LogExporter return Task.CompletedTask; } - public async Task QueryLogsAsync(long pageNumber, int entriesPerPage, bool descendingOrder, DateTime? start, DateTime? end, IPAddress clientIpAddress, DnsTransportProtocol? protocol, DnsServerResponseType? responseType, DnsResponseCode? rcode, string qname, DnsResourceRecordType? qtype, DnsClass? qclass) + public Task QueryLogsAsync(long pageNumber, int entriesPerPage, bool descendingOrder, DateTime? start, DateTime? end, IPAddress clientIpAddress, DnsTransportProtocol? protocol, DnsServerResponseType? responseType, DnsResponseCode? rcode, string qname, DnsResourceRecordType? qtype, DnsClass? qclass) { - return await Task.FromResult(new DnsLogPage(0, 0, 0, _emptyList)); + return Task.FromResult(new DnsLogPage(0, 0, 0, _emptyList)); } #endregion public #region private - private async Task ExportLogsAsync(CancellationToken cancellationToken = default) + private async Task ExportLogsAsync() + { + var logs = new List(BULK_INSERT_COUNT); + + // Process logs within the timer interval, then let the timer reschedule + while (logs.Count <= BULK_INSERT_COUNT && _logBuffer.TryTake(out var log)) + { + logs.Add(log); + } + + // If we have any logs to process, export them + if (logs.Count > 0) + { + await _exportManager.ImplementStrategyForAsync(logs); + } + } + + private async void HandleExportLogCallback(object? state) { try { - var logs = new List(BULK_INSERT_COUNT); - - while (!cancellationToken.IsCancellationRequested) - { - while ((logs.Count < BULK_INSERT_COUNT) && _logBuffer.TryTake(out LogEntry? log)) - { - if (log != null) - logs.Add(log); - } - - if (logs.Count > 0) - { - await _exportManager.ImplementStrategyForAsync(logs, cancellationToken); - - logs.Clear(); - } - } + await ExportLogsAsync().ConfigureAwait(false); } catch (Exception ex) { _dnsServer?.WriteLog(ex); } + finally + { + try + { + _queueTimer.Change(QUEUE_TIMER_INTERVAL, Timeout.Infinite); + } + catch (ObjectDisposedException) + { } + } } private void RegisterExportTargets() diff --git a/Apps/LogExporterApp/BufferManagementConfig.cs b/Apps/LogExporterApp/BufferManagementConfig.cs index 4ef64430..ca70e1e7 100644 --- a/Apps/LogExporterApp/BufferManagementConfig.cs +++ b/Apps/LogExporterApp/BufferManagementConfig.cs @@ -40,7 +40,7 @@ namespace LogExporter // Load configuration from JSON public static BufferManagementConfig? Deserialize(string json) { - return JsonSerializer.Deserialize(json); + return JsonSerializer.Deserialize(json, DnsConfigSerializerOptions.Default); } } @@ -79,4 +79,18 @@ namespace LogExporter [JsonPropertyName("headers")] public Dictionary? Headers { get; set; } } + + // Setup reusable options with a single instance + public static class DnsConfigSerializerOptions + { + public static readonly JsonSerializerOptions Default = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // Convert properties to camelCase + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, // For safe encoding + NumberHandling = JsonNumberHandling.Strict, + AllowTrailingCommas = true, // Allow trailing commas in JSON + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, // Convert dictionary keys to camelCase + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull // Ignore null values + }; + } } diff --git a/Apps/LogExporterApp/GrowableBuffer.cs b/Apps/LogExporterApp/GrowableBuffer.cs new file mode 100644 index 00000000..71504ff4 --- /dev/null +++ b/Apps/LogExporterApp/GrowableBuffer.cs @@ -0,0 +1,132 @@ +using System; +using System.Buffers; + +namespace LogExporter +{ + public class GrowableBuffer : IBufferWriter, IDisposable + { + // Gets the current length of the buffer contents + public int Length => _position; + + // Initial capacity to be used in the constructor + private const int DefaultInitialCapacity = 256; + + private Memory _buffer; + + private int _position; + + private bool disposedValue; + + public GrowableBuffer(int initialCapacity = DefaultInitialCapacity) + { + _buffer = new Memory(ArrayPool.Shared.Rent(initialCapacity)); + _position = 0; + } + + // IBufferWriter implementation + public void Advance(int count) + { + if (count < 0 || _position + count > _buffer.Length) + throw new ArgumentOutOfRangeException(nameof(count)); + + _position += count; + } + + // Appends a single element to the buffer + public void Append(T item) + { + EnsureCapacity(1); + _buffer.Span[_position++] = item; + } + + // Appends a span of elements to the buffer + public void Append(ReadOnlySpan span) + { + EnsureCapacity(span.Length); + span.CopyTo(_buffer.Span[_position..]); + _position += span.Length; + } + + // Clears the buffer for reuse without reallocating + public void Clear() => _position = 0; + + public Memory GetMemory(int sizeHint = 0) + { + EnsureCapacity(sizeHint); + return _buffer[_position..]; + } + + public Span GetSpan(int sizeHint = 0) + { + EnsureCapacity(sizeHint); + return _buffer.Span[_position..]; + } + + // Returns the buffer contents as an array + public T[] ToArray() + { + T[] result = new T[_position]; + _buffer.Span[.._position].CopyTo(result); + return result; + } + + // Returns the buffer contents as a ReadOnlySpan + public ReadOnlySpan ToSpan() => _buffer.Span[.._position]; + + public override string ToString() => _buffer.Span[.._position].ToString(); + + // Ensures the buffer has enough capacity to add more elements + private void EnsureCapacity(int additionalCapacity) + { + if (_position + additionalCapacity > _buffer.Length) + { + GrowBuffer(_position + additionalCapacity); + } + } + + // Grows the buffer to accommodate the required capacity + private void GrowBuffer(int requiredCapacity) + { + int newCapacity = Math.Max(_buffer.Length * 2, requiredCapacity); + + // Rent a larger buffer from the pool + T[] newArray = ArrayPool.Shared.Rent(newCapacity); + Memory newBuffer = new Memory(newArray); + + // Copy current contents to the new buffer + _buffer.Span[.._position].CopyTo(newBuffer.Span); + + // Return old buffer to the pool + ArrayPool.Shared.Return(_buffer.ToArray()); + + // Assign the new buffer + _buffer = newBuffer; + } + + #region IDisposable + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + ArrayPool.Shared.Return(_buffer.ToArray()); + _buffer = Memory.Empty; + _position = 0; + } + } + + disposedValue = true; + } + + #endregion IDisposable + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs index 1cc081e1..8d185831 100644 --- a/Apps/LogExporterApp/LogEntry.cs +++ b/Apps/LogExporterApp/LogEntry.cs @@ -1,11 +1,11 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text.Json.Serialization; +using System.Text; using System.Text.Json; using TechnitiumLibrary.Net.Dns; using TechnitiumLibrary.Net.Dns.ResourceRecords; -using System.Collections.Generic; namespace LogExporter { @@ -43,7 +43,7 @@ namespace LogExporter QuestionName = questionRecord.Name, QuestionType = questionRecord.Type, QuestionClass = questionRecord.Class, - Size = questionRecord.UncompressedLength + Size = questionRecord.UncompressedLength, })); } @@ -58,7 +58,7 @@ namespace LogExporter RecordClass = record.Class, RecordTtl = record.TTL, Size = record.UncompressedLength, - DnssecStatus = record.DnssecStatus + DnssecStatus = record.DnssecStatus, })); } @@ -73,11 +73,6 @@ namespace LogExporter } } - public override string ToString() - { - return JsonSerializer.Serialize(this, DnsLogSerializerOptions.Default); - } - public class Question { public string QuestionName { get; set; } @@ -95,34 +90,74 @@ namespace LogExporter public int Size { get; set; } public DnssecStatus DnssecStatus { get; set; } } - } - // Custom DateTime converter to handle UTC serialization in ISO 8601 format - public class JsonDateTimeConverter : JsonConverter - { - public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public ReadOnlySpan AsSpan() { - var dts = reader.GetString(); - return dts == null ? DateTime.MinValue : DateTime.Parse(dts); + // Initialize a ValueStringBuilder with some initial capacity + var buffer = new GrowableBuffer(256); + + using var writer = new Utf8JsonWriter(buffer); + + // Manually serialize the LogEntry as JSON + writer.WriteStartObject(); + + writer.WriteString("timestamp", Timestamp.ToUniversalTime().ToString("O")); + writer.WriteString("clientIp", ClientIp); + writer.WriteNumber("clientPort", ClientPort); + writer.WriteBoolean("dnssecOk", DnssecOk); + writer.WriteString("protocol", Protocol.ToString()); + writer.WriteString("responseCode", ResponseCode.ToString()); + + // Write Questions array + writer.WriteStartArray("questions"); + foreach (var question in Questions) + { + writer.WriteStartObject(); + writer.WriteString("questionName", question.QuestionName); + writer.WriteString("questionType", question.QuestionType.ToString()); + writer.WriteString("questionClass", question.QuestionClass.ToString()); + writer.WriteNumber("size", question.Size); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + + // Write Answers array (if exists) + if (Answers != null && Answers.Count > 0) + { + writer.WriteStartArray("answers"); + foreach (var answer in Answers) + { + writer.WriteStartObject(); + writer.WriteString("recordType", answer.RecordType.ToString()); + writer.WriteString("recordData", answer.RecordData); + writer.WriteString("recordClass", answer.RecordClass.ToString()); + writer.WriteNumber("recordTtl", answer.RecordTtl); + writer.WriteNumber("size", answer.Size); + writer.WriteString("dnssecStatus", answer.DnssecStatus.ToString()); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + writer.Flush(); + + return ConvertBytesToChars(buffer.ToSpan()); } - public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + public static Span ConvertBytesToChars(ReadOnlySpan byteSpan) { - writer.WriteStringValue(value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ")); - } - } + // Calculate the maximum required length for the char array + int maxCharCount = Encoding.UTF8.GetCharCount(byteSpan); - // Setup reusable options with a single instance - public static class DnsLogSerializerOptions - { - public static readonly JsonSerializerOptions Default = new JsonSerializerOptions - { - WriteIndented = false, // Newline delimited logs should not be multiline - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // Convert properties to camelCase - Converters = { new JsonStringEnumConverter(), new JsonDateTimeConverter() }, // Handle enums and DateTime conversion - Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, // For safe encoding - NumberHandling = JsonNumberHandling.Strict, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull // Ignore null values - }; - } + // Allocate a char array large enough to hold the converted characters + char[] charArray = new char[maxCharCount]; + + // Decode the byteSpan into the char array + int actualCharCount = Encoding.UTF8.GetChars(byteSpan, charArray); + + // Return a span of only the relevant portion of the char array + return new Span(charArray, 0, actualCharCount); + } + }; } diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs index 64aac98a..df70359f 100644 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -19,7 +19,6 @@ along with this program. If not, see . using System; using System.Collections.Generic; -using System.Threading; using System.Threading.Tasks; namespace LogExporter.Strategy @@ -43,25 +42,24 @@ namespace LogExporter.Strategy #region public - public IExportStrategy? GetStrategy() where T : IExportStrategy - { - _exportStrategies.TryGetValue(typeof(T), out var strategy); - return strategy; - } - - public async Task ImplementStrategyForAsync(List logs, CancellationToken cancellationToken = default) - { - foreach (var strategy in _exportStrategies.Values) - { - await strategy.ExportLogsAsync(logs, cancellationToken); - } - } - public void AddOrReplaceStrategy(IExportStrategy strategy) { _exportStrategies[strategy.GetType()] = strategy; } + public bool HasStrategy() + { + return _exportStrategies.Count > 0; + } + + public async Task ImplementStrategyForAsync(List logs) + { + foreach (var strategy in _exportStrategies.Values) + { + await strategy.ExportAsync(logs).ConfigureAwait(false); + } + } + #endregion public } } \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 8b4d6d6d..605012fa 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -17,16 +17,18 @@ along with this program. If not, see . */ +using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; namespace LogExporter.Strategy { - public class FileExportStrategy : IExportStrategy + public partial class FileExportStrategy : IExportStrategy { + #region variables private static readonly SemaphoreSlim _fileSemaphore = new SemaphoreSlim(1, 1); @@ -35,6 +37,7 @@ namespace LogExporter.Strategy private bool disposedValue; + #endregion variables #region constructor @@ -42,33 +45,46 @@ namespace LogExporter.Strategy public FileExportStrategy(string filePath) { _filePath = filePath; + } #endregion constructor #region public - public async Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) + public Task ExportAsync(List logs) { - var jsonLogs = new StringBuilder(logs.Count); + var buffer = new GrowableBuffer(); foreach (var log in logs) { - jsonLogs.AppendLine(log.ToString()); + buffer.Append(log.AsSpan()); + buffer.Append('\n'); } + Flush(buffer.ToSpan()); + return Task.CompletedTask; + } + private void Flush(ReadOnlySpan jsonLogs) + { // Wait to enter the semaphore - await _fileSemaphore.WaitAsync(cancellationToken); + _fileSemaphore.Wait(); try { // Use a FileStream with exclusive access - using var fileStream = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.None); - using var writer = new StreamWriter(fileStream); - await writer.WriteAsync(jsonLogs.ToString()); + var fileStream = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.Write); + var writer = new StreamWriter(fileStream); + writer.Write(jsonLogs); + writer.Close(); + fileStream.Dispose(); + } + catch (Exception ex) + { + Debug.WriteLine(ex); } finally { // Release the semaphore - _fileSemaphore.Release(); + _ = _fileSemaphore.Release(); } } @@ -96,6 +112,7 @@ namespace LogExporter.Strategy disposedValue = true; } } + #endregion IDisposable } } \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index a3d8d769..3967f53a 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -21,7 +21,6 @@ using System; using System.Collections.Generic; using System.Net.Http; using System.Text; -using System.Threading; using System.Threading.Tasks; namespace LogExporter.Strategy @@ -56,18 +55,20 @@ namespace LogExporter.Strategy #region public - public async Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) + public async Task ExportAsync(List logs) { - var jsonLogs = new StringBuilder(logs.Count); + var buffer = new GrowableBuffer(); foreach (var log in logs) { - jsonLogs.AppendLine(log.ToString()); + buffer.Append(log.AsSpan()); + buffer.Append('\n'); } + var content = buffer.ToString() ?? string.Empty; var request = new HttpRequestMessage { RequestUri = new Uri(_endpoint), Method = new HttpMethod(_method), - Content = new StringContent(jsonLogs.ToString(), Encoding.UTF8, "application/json") + Content = new StringContent( content, Encoding.UTF8, "application/json") }; if (_headers != null) @@ -78,7 +79,7 @@ namespace LogExporter.Strategy } } - var response = await _httpClient.SendAsync(request, cancellationToken); + var response = await _httpClient.SendAsync(request); if (!response.IsSuccessStatusCode) { throw new Exception($"Failed to export logs to {_endpoint}: {response.StatusCode}"); diff --git a/Apps/LogExporterApp/Strategy/IExportStrategy.cs b/Apps/LogExporterApp/Strategy/IExportStrategy.cs index c8aae94d..63feef3b 100644 --- a/Apps/LogExporterApp/Strategy/IExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/IExportStrategy.cs @@ -19,7 +19,6 @@ along with this program. If not, see . using System; using System.Collections.Generic; -using System.Threading; using System.Threading.Tasks; namespace LogExporter.Strategy @@ -29,6 +28,6 @@ namespace LogExporter.Strategy /// public interface IExportStrategy: IDisposable { - Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default); + Task ExportAsync(List logs); } } diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index 303ab2c1..a083392c 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -23,7 +23,6 @@ using SyslogNet.Client.Transport; using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using System.Threading.Tasks; namespace LogExporter.Strategy @@ -79,14 +78,13 @@ namespace LogExporter.Strategy #region public - public Task ExportLogsAsync(List logs, CancellationToken cancellationToken = default) + public Task ExportAsync(List logs) { return Task.Run(() => { var messages = new List(logs.Select(Convert)); _sender.Send(messages, _serializer); - } - , cancellationToken); + }); } #endregion public @@ -120,7 +118,7 @@ namespace LogExporter.Strategy private SyslogMessage Convert(LogEntry log) { // Create the structured data with all key details from LogEntry - var elements = new StructuredDataElement(_sdId, new Dictionary + var elements = new StructuredDataElement(_sdId, new Dictionary(StringComparer.OrdinalIgnoreCase) { { "timestamp", log.Timestamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ") }, { "clientIp", log.ClientIp }, @@ -173,17 +171,16 @@ namespace LogExporter.Strategy string questionSummary = log.Questions?.Count > 0 ? string.Join("; ", log.Questions.Select(q => $"QNAME: {q.QuestionName}; QTYPE: {q.QuestionType?.ToString() ?? "unknown"}; QCLASS: {q.QuestionClass?.ToString() ?? "unknown"}")) - : "No Questions"; + : string.Empty; // Build the answer summary in the desired format string answerSummary = log.Answers?.Count > 0 ? string.Join(", ", log.Answers.Select(a => a.RecordData)) - : "No Answers"; + : string.Empty; // Construct the message summary string to match the desired format string messageSummary = $"{questionSummary}; RCODE: {log.ResponseCode}; ANSWER: [{answerSummary}]"; - // Create and return the syslog message return new SyslogMessage( log.Timestamp, diff --git a/Apps/LogExporterApp/dnsApp.config b/Apps/LogExporterApp/dnsApp.config index beb75381..36ccefc0 100644 --- a/Apps/LogExporterApp/dnsApp.config +++ b/Apps/LogExporterApp/dnsApp.config @@ -2,7 +2,7 @@ "maxLogEntries": 1000, "file": { "path": "/var/log/dns_logs.json", - "enabled": true + "enabled": false }, "http": { "endpoint": "http://example.com/logs", @@ -10,12 +10,12 @@ "headers": { "Authorization": "Bearer abc123" }, - "enabled": true + "enabled": false }, "syslog": { "address": "127.0.0.1", "port": 514, "protocol": "UDP", - "enabled": true + "enabled": false } } \ No newline at end of file From 54952f19778fa6830a7f37fa5c6d19566faa284f Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Sun, 20 Oct 2024 17:44:46 +0300 Subject: [PATCH 07/17] Simplified JSON serialization. While there was lower memory allocation, the speed was an issue. --- Apps/LogExporterApp/GrowableBuffer.cs | 132 ------------------ Apps/LogExporterApp/LogEntry.cs | 94 ++++--------- .../Strategy/FileExportStrategy.cs | 10 +- .../Strategy/HttpExportStrategy.cs | 7 +- .../Strategy/SyslogExportStrategy.cs | 4 +- 5 files changed, 40 insertions(+), 207 deletions(-) delete mode 100644 Apps/LogExporterApp/GrowableBuffer.cs diff --git a/Apps/LogExporterApp/GrowableBuffer.cs b/Apps/LogExporterApp/GrowableBuffer.cs deleted file mode 100644 index 71504ff4..00000000 --- a/Apps/LogExporterApp/GrowableBuffer.cs +++ /dev/null @@ -1,132 +0,0 @@ -using System; -using System.Buffers; - -namespace LogExporter -{ - public class GrowableBuffer : IBufferWriter, IDisposable - { - // Gets the current length of the buffer contents - public int Length => _position; - - // Initial capacity to be used in the constructor - private const int DefaultInitialCapacity = 256; - - private Memory _buffer; - - private int _position; - - private bool disposedValue; - - public GrowableBuffer(int initialCapacity = DefaultInitialCapacity) - { - _buffer = new Memory(ArrayPool.Shared.Rent(initialCapacity)); - _position = 0; - } - - // IBufferWriter implementation - public void Advance(int count) - { - if (count < 0 || _position + count > _buffer.Length) - throw new ArgumentOutOfRangeException(nameof(count)); - - _position += count; - } - - // Appends a single element to the buffer - public void Append(T item) - { - EnsureCapacity(1); - _buffer.Span[_position++] = item; - } - - // Appends a span of elements to the buffer - public void Append(ReadOnlySpan span) - { - EnsureCapacity(span.Length); - span.CopyTo(_buffer.Span[_position..]); - _position += span.Length; - } - - // Clears the buffer for reuse without reallocating - public void Clear() => _position = 0; - - public Memory GetMemory(int sizeHint = 0) - { - EnsureCapacity(sizeHint); - return _buffer[_position..]; - } - - public Span GetSpan(int sizeHint = 0) - { - EnsureCapacity(sizeHint); - return _buffer.Span[_position..]; - } - - // Returns the buffer contents as an array - public T[] ToArray() - { - T[] result = new T[_position]; - _buffer.Span[.._position].CopyTo(result); - return result; - } - - // Returns the buffer contents as a ReadOnlySpan - public ReadOnlySpan ToSpan() => _buffer.Span[.._position]; - - public override string ToString() => _buffer.Span[.._position].ToString(); - - // Ensures the buffer has enough capacity to add more elements - private void EnsureCapacity(int additionalCapacity) - { - if (_position + additionalCapacity > _buffer.Length) - { - GrowBuffer(_position + additionalCapacity); - } - } - - // Grows the buffer to accommodate the required capacity - private void GrowBuffer(int requiredCapacity) - { - int newCapacity = Math.Max(_buffer.Length * 2, requiredCapacity); - - // Rent a larger buffer from the pool - T[] newArray = ArrayPool.Shared.Rent(newCapacity); - Memory newBuffer = new Memory(newArray); - - // Copy current contents to the new buffer - _buffer.Span[.._position].CopyTo(newBuffer.Span); - - // Return old buffer to the pool - ArrayPool.Shared.Return(_buffer.ToArray()); - - // Assign the new buffer - _buffer = newBuffer; - } - - #region IDisposable - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - ArrayPool.Shared.Return(_buffer.ToArray()); - _buffer = Memory.Empty; - _position = 0; - } - } - - disposedValue = true; - } - - #endregion IDisposable - } -} \ No newline at end of file diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs index 8d185831..c7d1f9f4 100644 --- a/Apps/LogExporterApp/LogEntry.cs +++ b/Apps/LogExporterApp/LogEntry.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using TechnitiumLibrary.Net.Dns; using TechnitiumLibrary.Net.Dns.ResourceRecords; @@ -91,73 +92,38 @@ namespace LogExporter public DnssecStatus DnssecStatus { get; set; } } - public ReadOnlySpan AsSpan() + public override string ToString() { - // Initialize a ValueStringBuilder with some initial capacity - var buffer = new GrowableBuffer(256); - - using var writer = new Utf8JsonWriter(buffer); - - // Manually serialize the LogEntry as JSON - writer.WriteStartObject(); - - writer.WriteString("timestamp", Timestamp.ToUniversalTime().ToString("O")); - writer.WriteString("clientIp", ClientIp); - writer.WriteNumber("clientPort", ClientPort); - writer.WriteBoolean("dnssecOk", DnssecOk); - writer.WriteString("protocol", Protocol.ToString()); - writer.WriteString("responseCode", ResponseCode.ToString()); - - // Write Questions array - writer.WriteStartArray("questions"); - foreach (var question in Questions) - { - writer.WriteStartObject(); - writer.WriteString("questionName", question.QuestionName); - writer.WriteString("questionType", question.QuestionType.ToString()); - writer.WriteString("questionClass", question.QuestionClass.ToString()); - writer.WriteNumber("size", question.Size); - writer.WriteEndObject(); - } - writer.WriteEndArray(); - - // Write Answers array (if exists) - if (Answers != null && Answers.Count > 0) - { - writer.WriteStartArray("answers"); - foreach (var answer in Answers) - { - writer.WriteStartObject(); - writer.WriteString("recordType", answer.RecordType.ToString()); - writer.WriteString("recordData", answer.RecordData); - writer.WriteString("recordClass", answer.RecordClass.ToString()); - writer.WriteNumber("recordTtl", answer.RecordTtl); - writer.WriteNumber("size", answer.Size); - writer.WriteString("dnssecStatus", answer.DnssecStatus.ToString()); - writer.WriteEndObject(); - } - writer.WriteEndArray(); - } - - writer.WriteEndObject(); - writer.Flush(); - - return ConvertBytesToChars(buffer.ToSpan()); + return JsonSerializer.Serialize(this, DnsLogSerializerOptions.Default); } - public static Span ConvertBytesToChars(ReadOnlySpan byteSpan) + // Custom DateTime converter to handle UTC serialization in ISO 8601 format + public class JsonDateTimeConverter : JsonConverter { - // Calculate the maximum required length for the char array - int maxCharCount = Encoding.UTF8.GetCharCount(byteSpan); + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dts = reader.GetString(); + return dts == null ? DateTime.MinValue : DateTime.Parse(dts); + } - // Allocate a char array large enough to hold the converted characters - char[] charArray = new char[maxCharCount]; - - // Decode the byteSpan into the char array - int actualCharCount = Encoding.UTF8.GetChars(byteSpan, charArray); - - // Return a span of only the relevant portion of the char array - return new Span(charArray, 0, actualCharCount); + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ")); + } } - }; -} + + // Setup reusable options with a single instance + public static class DnsLogSerializerOptions + { + public static readonly JsonSerializerOptions Default = new JsonSerializerOptions + { + WriteIndented = false, // Newline delimited logs should not be multiline + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // Convert properties to camelCase + Converters = { new JsonStringEnumConverter(), new JsonDateTimeConverter() }, // Handle enums and DateTime conversion + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, // For safe encoding + NumberHandling = JsonNumberHandling.Strict, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull // Ignore null values + }; + } + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 605012fa..351e0a13 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -21,6 +21,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -54,17 +55,16 @@ namespace LogExporter.Strategy public Task ExportAsync(List logs) { - var buffer = new GrowableBuffer(); + var jsonLogs = new StringBuilder(logs.Count); foreach (var log in logs) { - buffer.Append(log.AsSpan()); - buffer.Append('\n'); + jsonLogs.AppendLine(log.ToString()); } - Flush(buffer.ToSpan()); + Flush(jsonLogs.ToString()); return Task.CompletedTask; } - private void Flush(ReadOnlySpan jsonLogs) + private void Flush(string jsonLogs) { // Wait to enter the semaphore _fileSemaphore.Wait(); diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index 3967f53a..46722017 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -57,13 +57,12 @@ namespace LogExporter.Strategy public async Task ExportAsync(List logs) { - var buffer = new GrowableBuffer(); + var jsonLogs = new StringBuilder(logs.Count); foreach (var log in logs) { - buffer.Append(log.AsSpan()); - buffer.Append('\n'); + jsonLogs.AppendLine(log.ToString()); } - var content = buffer.ToString() ?? string.Empty; + var content = jsonLogs.ToString() ?? string.Empty; var request = new HttpRequestMessage { RequestUri = new Uri(_endpoint), diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index a083392c..a01c81c6 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -129,7 +129,7 @@ namespace LogExporter.Strategy }); // Add each question to the structured data - if (log.Questions != null && log.Questions.Count > 0) + if (log.Questions?.Count > 0) { for (int i = 0; i < log.Questions.Count; i++) { @@ -142,7 +142,7 @@ namespace LogExporter.Strategy } // Add each answer to the structured data - if (log.Answers != null && log.Answers.Count > 0) + if (log.Answers?.Count > 0) { for (int i = 0; i < log.Answers.Count; i++) { From ebb5908dc4fd6489f2e6046681058c09d0e9e2e4 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Sun, 20 Oct 2024 17:57:59 +0300 Subject: [PATCH 08/17] Improved async handlin when writing to files --- .../Strategy/FileExportStrategy.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 351e0a13..e44ab627 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -60,22 +60,21 @@ namespace LogExporter.Strategy { jsonLogs.AppendLine(log.ToString()); } - Flush(jsonLogs.ToString()); - return Task.CompletedTask; + return FlushAsync(jsonLogs.ToString()); } - private void Flush(string jsonLogs) + private async Task FlushAsync(string jsonLogs) { // Wait to enter the semaphore - _fileSemaphore.Wait(); + await _fileSemaphore.WaitAsync(); try { // Use a FileStream with exclusive access - var fileStream = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.Write); - var writer = new StreamWriter(fileStream); - writer.Write(jsonLogs); - writer.Close(); - fileStream.Dispose(); + using (var fileStream = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.None)) + using (var writer = new StreamWriter(fileStream)) + { + await writer.WriteAsync(jsonLogs); + } } catch (Exception ex) { @@ -83,8 +82,11 @@ namespace LogExporter.Strategy } finally { - // Release the semaphore - _ = _fileSemaphore.Release(); + // Ensure semaphore is released only if it was successfully acquired + if (_fileSemaphore.CurrentCount == 0) + { + _fileSemaphore.Release(); + } } } From 82e116d38f9a9502d0cf895bdeddfc5545136148 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Sun, 20 Oct 2024 18:02:42 +0300 Subject: [PATCH 09/17] Fixed initial StringBuilder size --- Apps/LogExporterApp/Strategy/FileExportStrategy.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index e44ab627..7783ecb0 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -27,9 +27,8 @@ using System.Threading.Tasks; namespace LogExporter.Strategy { - public partial class FileExportStrategy : IExportStrategy + public class FileExportStrategy : IExportStrategy { - #region variables private static readonly SemaphoreSlim _fileSemaphore = new SemaphoreSlim(1, 1); @@ -38,7 +37,6 @@ namespace LogExporter.Strategy private bool disposedValue; - #endregion variables #region constructor @@ -46,7 +44,6 @@ namespace LogExporter.Strategy public FileExportStrategy(string filePath) { _filePath = filePath; - } #endregion constructor @@ -55,7 +52,7 @@ namespace LogExporter.Strategy public Task ExportAsync(List logs) { - var jsonLogs = new StringBuilder(logs.Count); + var jsonLogs = new StringBuilder(logs.Count * 250); foreach (var log in logs) { jsonLogs.AppendLine(log.ToString()); @@ -107,7 +104,11 @@ namespace LogExporter.Strategy { if (disposing) { - _fileSemaphore.Release(); + // Ensure semaphore is released only if it was successfully acquired + if (_fileSemaphore.CurrentCount == 0) + { + _fileSemaphore.Release(); + } _fileSemaphore.Dispose(); } From 2220161e87a359a8bf3dc045b06cc0d17c1da719 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 20 Nov 2024 14:30:35 +0200 Subject: [PATCH 10/17] Implemented syslog with Serilog --- Apps/LogExporterApp/LogExporterApp.csproj | 4 +- .../Strategy/SyslogExportStrategy.cs | 145 +++++++++--------- 2 files changed, 77 insertions(+), 72 deletions(-) diff --git a/Apps/LogExporterApp/LogExporterApp.csproj b/Apps/LogExporterApp/LogExporterApp.csproj index 6c6107d2..16020dbf 100644 --- a/Apps/LogExporterApp/LogExporterApp.csproj +++ b/Apps/LogExporterApp/LogExporterApp.csproj @@ -20,7 +20,9 @@ - + + + diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index a01c81c6..eee2d356 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -17,9 +17,10 @@ along with this program. If not, see . */ -using SyslogNet.Client; -using SyslogNet.Client.Serialization; -using SyslogNet.Client.Transport; +using Serilog; +using Serilog.Events; +using Serilog.Parsing; +using Serilog.Sinks.Syslog; using System; using System.Collections.Generic; using System.Linq; @@ -33,21 +34,17 @@ namespace LogExporter.Strategy private const string _appName = "Technitium DNS Server"; - private const string _msgId = "dnslog"; - - private const string _sdId = "dnsparams"; + private const string _sdId = "meta"; private const string DEFAUL_PROTOCOL = "udp"; private const int DEFAULT_PORT = 514; - private readonly string _host; + private readonly Facility _facility = Facility.Local6; - private readonly string _processId; + private readonly Rfc5424Formatter _formatter; - private readonly ISyslogMessageSender _sender; - - private readonly ISyslogMessageSerializer _serializer; + private readonly Serilog.Core.Logger _sender; private bool disposedValue; @@ -60,18 +57,18 @@ namespace LogExporter.Strategy port ??= DEFAULT_PORT; protocol ??= DEFAUL_PROTOCOL; + var conf = new LoggerConfiguration(); + _sender = protocol.ToLowerInvariant() switch { - "tls" => new SyslogEncryptedTcpSender(address, port.Value), - "tcp" => new SyslogTcpSender(address, port.Value), - "udp" => new SyslogUdpSender(address, port.Value), - "local" => new SyslogLocalSender(), + "tls" => conf.WriteTo.TcpSyslog(address, port.Value, _appName, FramingType.OCTET_COUNTING, SyslogFormat.RFC5424, _facility, useTls: true).Enrich.FromLogContext().CreateLogger(), + "tcp" => conf.WriteTo.TcpSyslog(address, port.Value, _appName, FramingType.OCTET_COUNTING, SyslogFormat.RFC5424, _facility, useTls: false).Enrich.FromLogContext().CreateLogger(), + "udp" => conf.WriteTo.UdpSyslog(address, port.Value, _appName, SyslogFormat.RFC5424, _facility).Enrich.FromLogContext().CreateLogger(), + "local" => conf.WriteTo.LocalSyslog(_appName, _facility).Enrich.FromLogContext().CreateLogger(), _ => throw new Exception("Invalid protocol specified"), }; - _serializer = new SyslogRfc5424MessageSerializer(); - _processId = Environment.ProcessId.ToString(); - _host = Environment.MachineName; + _formatter = new Rfc5424Formatter(_facility, _appName, null, _sdId, Environment.MachineName); } #endregion constructor @@ -80,11 +77,12 @@ namespace LogExporter.Strategy public Task ExportAsync(List logs) { - return Task.Run(() => - { - var messages = new List(logs.Select(Convert)); - _sender.Send(messages, _serializer); - }); + var tasks = logs + .Select(log => Task.Run(() => + Log.Information(_formatter.FormatMessage(Convert(log)))) + ); + + return Task.WhenAll(tasks); } #endregion public @@ -115,83 +113,88 @@ namespace LogExporter.Strategy #region private - private SyslogMessage Convert(LogEntry log) + private LogEvent Convert(LogEntry log) { - // Create the structured data with all key details from LogEntry - var elements = new StructuredDataElement(_sdId, new Dictionary(StringComparer.OrdinalIgnoreCase) + // Initialize properties with base log details + var properties = new List { - { "timestamp", log.Timestamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ") }, - { "clientIp", log.ClientIp }, - { "clientPort", log.ClientPort.ToString() }, - { "dnssecOk", log.DnssecOk.ToString() }, - { "protocol", log.Protocol.ToString() }, - { "rCode", log.ResponseCode.ToString() } - }); + new LogEventProperty("timestamp", new ScalarValue(log.Timestamp.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))), + new LogEventProperty("clientIp", new ScalarValue(log.ClientIp)), + new LogEventProperty("clientPort", new ScalarValue(log.ClientPort.ToString())), + new LogEventProperty("dnssecOk", new ScalarValue(log.DnssecOk.ToString())), + new LogEventProperty("protocol", new ScalarValue(log.Protocol.ToString())), + new LogEventProperty("rCode", new ScalarValue(log.ResponseCode.ToString())) + }; - // Add each question to the structured data + // Add each question as properties if (log.Questions?.Count > 0) { for (int i = 0; i < log.Questions.Count; i++) { var question = log.Questions[i]; - elements.Parameters.Add($"qName_{i}", question.QuestionName); - elements.Parameters.Add($"qType_{i}", question.QuestionType.HasValue ? question.QuestionType.Value.ToString() : "unknown"); - elements.Parameters.Add($"qClass_{i}", question.QuestionClass.HasValue ? question.QuestionClass.Value.ToString() : "unknown"); - elements.Parameters.Add($"qSize_{i}", question.Size.ToString()); + properties.Add(new LogEventProperty($"qName_{i}", new ScalarValue(question.QuestionName))); + properties.Add(new LogEventProperty($"qType_{i}", new ScalarValue(question.QuestionType?.ToString() ?? "unknown"))); + properties.Add(new LogEventProperty($"qClass_{i}", new ScalarValue(question.QuestionClass?.ToString() ?? "unknown"))); + properties.Add(new LogEventProperty($"qSize_{i}", new ScalarValue(question.Size.ToString()))); } + + // Generate questions summary + var questionSummary = string.Join("; ", log.Questions.Select((q, i) => + $"QNAME_{i}: {q.QuestionName}, QTYPE: {q.QuestionType?.ToString() ?? "unknown"}, QCLASS: {q.QuestionClass?.ToString() ?? "unknown"}")); + properties.Add(new LogEventProperty("questionsSummary", new ScalarValue(questionSummary))); + } + else + { + properties.Add(new LogEventProperty("questionsSummary", new ScalarValue(string.Empty))); } - // Add each answer to the structured data + // Add each answer as properties if (log.Answers?.Count > 0) { for (int i = 0; i < log.Answers.Count; i++) { var answer = log.Answers[i]; - elements.Parameters.Add($"aType_{i}", answer.RecordType.ToString()); - elements.Parameters.Add($"aData_{i}", answer.RecordData); - elements.Parameters.Add($"aClass_{i}", answer.RecordClass.ToString()); - elements.Parameters.Add($"aTtl_{i}", answer.RecordTtl.ToString()); - elements.Parameters.Add($"aSize_{i}", answer.Size.ToString()); - elements.Parameters.Add($"aDnssecStatus_{i}", answer.DnssecStatus.ToString()); + properties.Add(new LogEventProperty($"aType_{i}", new ScalarValue(answer.RecordType.ToString()))); + properties.Add(new LogEventProperty($"aData_{i}", new ScalarValue(answer.RecordData))); + properties.Add(new LogEventProperty($"aClass_{i}", new ScalarValue(answer.RecordClass.ToString()))); + properties.Add(new LogEventProperty($"aTtl_{i}", new ScalarValue(answer.RecordTtl.ToString()))); + properties.Add(new LogEventProperty($"aSize_{i}", new ScalarValue(answer.Size.ToString()))); + properties.Add(new LogEventProperty($"aDnssecStatus_{i}", new ScalarValue(answer.DnssecStatus.ToString()))); } + + // Generate answers summary + var answerSummary = string.Join(", ", log.Answers.Select(a => a.RecordData)); + properties.Add(new LogEventProperty("answersSummary", new ScalarValue(answerSummary))); + } + else + { + properties.Add(new LogEventProperty("answersSummary", new ScalarValue(string.Empty))); } - // Include request and response tags if present + // Add request and response tags if present if (log.RequestTag != null) { - elements.Parameters.Add("requestTag", log.RequestTag.ToString()); + properties.Add(new LogEventProperty("requestTag", new ScalarValue(log.RequestTag.ToString()))); } if (log.ResponseTag != null) { - elements.Parameters.Add("responseTag", log.ResponseTag.ToString()); + properties.Add(new LogEventProperty("responseTag", new ScalarValue(log.ResponseTag.ToString()))); } - // Build a comprehensive message summary - string questionSummary = log.Questions?.Count > 0 - ? string.Join("; ", log.Questions.Select(q => - $"QNAME: {q.QuestionName}; QTYPE: {q.QuestionType?.ToString() ?? "unknown"}; QCLASS: {q.QuestionClass?.ToString() ?? "unknown"}")) - : string.Empty; + // Define the message template to match the original summary format + const string templateText = "{questionsSummary}; RCODE: {rCode}; ANSWER: [{answersSummary}]"; - // Build the answer summary in the desired format - string answerSummary = log.Answers?.Count > 0 - ? string.Join(", ", log.Answers.Select(a => a.RecordData)) - : string.Empty; + // Parse the template + var template = new MessageTemplateParser().Parse(templateText); - // Construct the message summary string to match the desired format - string messageSummary = $"{questionSummary}; RCODE: {log.ResponseCode}; ANSWER: [{answerSummary}]"; - - // Create and return the syslog message - return new SyslogMessage( - log.Timestamp, - Facility.UserLevelMessages, - Severity.Informational, - _host, - _appName, - _processId, - _msgId, - messageSummary, - elements + // Create the LogEvent and return it + return new LogEvent( + timestamp: log.Timestamp, + level: LogEventLevel.Information, + exception: null, + messageTemplate: template, + properties: properties ); } From 1a9153dd74115b155cd95191090c34cdcef76971 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 20 Nov 2024 15:30:18 +0200 Subject: [PATCH 11/17] Implemented http logging with Serilog --- Apps/LogExporterApp/App.cs | 6 +- Apps/LogExporterApp/BufferManagementConfig.cs | 3 - Apps/LogExporterApp/LogExporterApp.csproj | 1 + .../Strategy/HttpExportStrategy.cs | 96 +++++++++++-------- Apps/LogExporterApp/dnsApp.config | 5 +- 5 files changed, 61 insertions(+), 50 deletions(-) diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index 5030de81..6220d29d 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -191,9 +191,9 @@ namespace LogExporter } // Register the different strategies using the helper - RegisterIfEnabled(_config.FileTarget, target => new FileExportStrategy(target.Path)); - RegisterIfEnabled(_config.HttpTarget, target => new HttpExportStrategy(target.Endpoint, target.Method, target.Headers)); - RegisterIfEnabled(_config.SyslogTarget, target => new SyslogExportStrategy(target.Address, target.Port, target.Protocol)); + RegisterIfEnabled(_config!.FileTarget!, target => new FileExportStrategy(target.Path)); + RegisterIfEnabled(_config!.HttpTarget!, target => new HttpExportStrategy(target.Endpoint, target.Headers)); + RegisterIfEnabled(_config!.SyslogTarget!, target => new SyslogExportStrategy(target.Address, target.Port, target.Protocol)); } #endregion private diff --git a/Apps/LogExporterApp/BufferManagementConfig.cs b/Apps/LogExporterApp/BufferManagementConfig.cs index ca70e1e7..76602953 100644 --- a/Apps/LogExporterApp/BufferManagementConfig.cs +++ b/Apps/LogExporterApp/BufferManagementConfig.cs @@ -73,9 +73,6 @@ namespace LogExporter [JsonPropertyName("endpoint")] public string Endpoint { get; set; } - [JsonPropertyName("method")] - public string Method { get; set; } - [JsonPropertyName("headers")] public Dictionary? Headers { get; set; } } diff --git a/Apps/LogExporterApp/LogExporterApp.csproj b/Apps/LogExporterApp/LogExporterApp.csproj index 16020dbf..e6235eb4 100644 --- a/Apps/LogExporterApp/LogExporterApp.csproj +++ b/Apps/LogExporterApp/LogExporterApp.csproj @@ -20,6 +20,7 @@ + diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index 46722017..8a5fb611 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -17,10 +17,15 @@ along with this program. If not, see . */ +using Microsoft.Extensions.Configuration; +using Serilog; +using Serilog.Sinks.Http; using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Net.Http; -using System.Text; +using System.Threading; using System.Threading.Tasks; namespace LogExporter.Strategy @@ -29,13 +34,7 @@ namespace LogExporter.Strategy { #region variables - private readonly string _endpoint; - - private readonly Dictionary? _headers; - - private readonly HttpClient _httpClient; - - private readonly string _method; + private readonly Serilog.Core.Logger _sender; private bool disposedValue; @@ -43,46 +42,27 @@ namespace LogExporter.Strategy #region constructor - public HttpExportStrategy(string endpoint, string method, Dictionary? headers) + public HttpExportStrategy(string endpoint, Dictionary? headers = null) { - _endpoint = endpoint; - _method = method; - _headers = headers; - _httpClient = new HttpClient(); + IConfigurationRoot? configuration = null; + if (headers != null) + { + configuration = new ConfigurationBuilder() + .AddInMemoryCollection(headers) + .Build(); + } + + _sender = new LoggerConfiguration().WriteTo.Http(endpoint, null, httpClient: new CustomHttpClient(), configuration: configuration).Enrich.FromLogContext().CreateLogger(); } #endregion constructor #region public - public async Task ExportAsync(List logs) + public Task ExportAsync(List logs) { - var jsonLogs = new StringBuilder(logs.Count); - foreach (var log in logs) - { - jsonLogs.AppendLine(log.ToString()); - } - var content = jsonLogs.ToString() ?? string.Empty; - var request = new HttpRequestMessage - { - RequestUri = new Uri(_endpoint), - Method = new HttpMethod(_method), - Content = new StringContent( content, Encoding.UTF8, "application/json") - }; - - if (_headers != null) - { - foreach (var header in _headers) - { - request.Headers.Add(header.Key, header.Value); - } - } - - var response = await _httpClient.SendAsync(request); - if (!response.IsSuccessStatusCode) - { - throw new Exception($"Failed to export logs to {_endpoint}: {response.StatusCode}"); - } + var tasks = logs.Select(log => Task.Run(() => _sender.Information(log.ToString()))); + return Task.WhenAll(tasks); } #endregion public @@ -102,7 +82,7 @@ namespace LogExporter.Strategy { if (disposing) { - _httpClient.Dispose(); + _sender.Dispose(); } disposedValue = true; @@ -110,5 +90,39 @@ namespace LogExporter.Strategy } #endregion IDisposable + + #region Classes + + public class CustomHttpClient : IHttpClient + { + private readonly HttpClient httpClient; + + public CustomHttpClient() => httpClient = new HttpClient(); + + public void Configure(IConfiguration configuration) + { + foreach (var pair in configuration.GetChildren()) + { + httpClient.DefaultRequestHeaders.Add(pair.Key, pair.Value); + } + } + + public void Dispose() + { + httpClient?.Dispose(); + } + + public async Task PostAsync(string requestUri, Stream contentStream, CancellationToken cancellationToken) + { + using var content = new StreamContent(contentStream); + content.Headers.Add("Content-Type", "application/json"); + + return await httpClient + .PostAsync(requestUri, content, cancellationToken) + .ConfigureAwait(false); + } + } + + #endregion Classes } } \ No newline at end of file diff --git a/Apps/LogExporterApp/dnsApp.config b/Apps/LogExporterApp/dnsApp.config index 36ccefc0..b6d177ea 100644 --- a/Apps/LogExporterApp/dnsApp.config +++ b/Apps/LogExporterApp/dnsApp.config @@ -1,12 +1,11 @@ { "maxLogEntries": 1000, "file": { - "path": "/var/log/dns_logs.json", + "path": "./dns_logs.json", "enabled": false }, "http": { - "endpoint": "http://example.com/logs", - "method": "POST", + "endpoint": "http://localhost:5000/logs", "headers": { "Authorization": "Bearer abc123" }, From c7b36a7b84d0fdbc48c944eb17828ab65e546330 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 20 Nov 2024 15:37:15 +0200 Subject: [PATCH 12/17] Implemented file logging with Serilog --- Apps/LogExporterApp/LogEntry.cs | 1 - .../Strategy/FileExportStrategy.cs | 55 +++---------------- 2 files changed, 7 insertions(+), 49 deletions(-) diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs index c7d1f9f4..ff360d5b 100644 --- a/Apps/LogExporterApp/LogEntry.cs +++ b/Apps/LogExporterApp/LogEntry.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using TechnitiumLibrary.Net.Dns; diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 7783ecb0..ed3652d0 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -17,12 +17,9 @@ along with this program. If not, see . */ -using System; +using Serilog; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Text; -using System.Threading; +using System.Linq; using System.Threading.Tasks; namespace LogExporter.Strategy @@ -31,9 +28,7 @@ namespace LogExporter.Strategy { #region variables - private static readonly SemaphoreSlim _fileSemaphore = new SemaphoreSlim(1, 1); - - private readonly string _filePath; + private readonly Serilog.Core.Logger _sender; private bool disposedValue; @@ -43,7 +38,7 @@ namespace LogExporter.Strategy public FileExportStrategy(string filePath) { - _filePath = filePath; + _sender = new LoggerConfiguration().WriteTo.File(filePath, outputTemplate:"{Message}{Newline}").CreateLogger(); } #endregion constructor @@ -52,39 +47,8 @@ namespace LogExporter.Strategy public Task ExportAsync(List logs) { - var jsonLogs = new StringBuilder(logs.Count * 250); - foreach (var log in logs) - { - jsonLogs.AppendLine(log.ToString()); - } - return FlushAsync(jsonLogs.ToString()); - } - - private async Task FlushAsync(string jsonLogs) - { - // Wait to enter the semaphore - await _fileSemaphore.WaitAsync(); - try - { - // Use a FileStream with exclusive access - using (var fileStream = new FileStream(_filePath, FileMode.Append, FileAccess.Write, FileShare.None)) - using (var writer = new StreamWriter(fileStream)) - { - await writer.WriteAsync(jsonLogs); - } - } - catch (Exception ex) - { - Debug.WriteLine(ex); - } - finally - { - // Ensure semaphore is released only if it was successfully acquired - if (_fileSemaphore.CurrentCount == 0) - { - _fileSemaphore.Release(); - } - } + var tasks = logs.Select(log => Task.Run(() => _sender.Information(log.ToString()))); + return Task.WhenAll(tasks); } #endregion public @@ -104,12 +68,7 @@ namespace LogExporter.Strategy { if (disposing) { - // Ensure semaphore is released only if it was successfully acquired - if (_fileSemaphore.CurrentCount == 0) - { - _fileSemaphore.Release(); - } - _fileSemaphore.Dispose(); + _sender.Dispose(); } disposedValue = true; From b4d90fca2bce5607d9a9af15914f65716156544b Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Wed, 20 Nov 2024 15:47:54 +0200 Subject: [PATCH 13/17] Fixed message formatting on file targets --- Apps/LogExporterApp/Strategy/FileExportStrategy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index ed3652d0..cedb65f4 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -38,7 +38,7 @@ namespace LogExporter.Strategy public FileExportStrategy(string filePath) { - _sender = new LoggerConfiguration().WriteTo.File(filePath, outputTemplate:"{Message}{Newline}").CreateLogger(); + _sender = new LoggerConfiguration().WriteTo.File(filePath, outputTemplate: "{Message:lj}{Newline}").CreateLogger(); } #endregion constructor From 3f67e7a050afd8a4d65b1a47053a9b3d89d298cc Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 21 Nov 2024 19:32:01 +0200 Subject: [PATCH 14/17] Removed Async code as we don't handle async ops anymore --- Apps/LogExporterApp/App.cs | 11 +++++------ Apps/LogExporterApp/LogExporterApp.csproj | 6 +++++- Apps/LogExporterApp/Strategy/ExportManager.cs | 4 ++-- Apps/LogExporterApp/Strategy/FileExportStrategy.cs | 10 +++++----- Apps/LogExporterApp/Strategy/HttpExportStrategy.cs | 9 +++++---- Apps/LogExporterApp/Strategy/IExportStrategy.cs | 2 +- Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs | 12 +++++------- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index 6220d29d..333c5b45 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -25,7 +25,6 @@ using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; -using TechnitiumLibrary; using TechnitiumLibrary.Net.Dns; using TechnitiumLibrary.Net.Dns.ResourceRecords; @@ -82,7 +81,7 @@ namespace LogExporter { _queueTimer?.Dispose(); - ExportLogsAsync().Sync(); //flush any pending logs + ExportLogs(); //flush any pending logs _logBuffer.Dispose(); } @@ -138,7 +137,7 @@ namespace LogExporter #region private - private async Task ExportLogsAsync() + private void ExportLogs() { var logs = new List(BULK_INSERT_COUNT); @@ -151,15 +150,15 @@ namespace LogExporter // If we have any logs to process, export them if (logs.Count > 0) { - await _exportManager.ImplementStrategyForAsync(logs); + _exportManager.ImplementStrategy(logs); } } - private async void HandleExportLogCallback(object? state) + private void HandleExportLogCallback(object? state) { try { - await ExportLogsAsync().ConfigureAwait(false); + ExportLogs(); } catch (Exception ex) { diff --git a/Apps/LogExporterApp/LogExporterApp.csproj b/Apps/LogExporterApp/LogExporterApp.csproj index e6235eb4..87ffa0d9 100644 --- a/Apps/LogExporterApp/LogExporterApp.csproj +++ b/Apps/LogExporterApp/LogExporterApp.csproj @@ -20,9 +20,10 @@ - + + @@ -33,6 +34,9 @@ + + Serilog.Sinks.SyslogMessages.dll + ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.Net.dll false diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs index df70359f..f6e2d59c 100644 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -52,11 +52,11 @@ namespace LogExporter.Strategy return _exportStrategies.Count > 0; } - public async Task ImplementStrategyForAsync(List logs) + public void ImplementStrategy(List logs) { foreach (var strategy in _exportStrategies.Values) { - await strategy.ExportAsync(logs).ConfigureAwait(false); + strategy.Export(logs); } } diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index cedb65f4..181bd1c8 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -19,8 +19,6 @@ along with this program. If not, see . using Serilog; using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; namespace LogExporter.Strategy { @@ -45,10 +43,12 @@ namespace LogExporter.Strategy #region public - public Task ExportAsync(List logs) + public void Export(List logs) { - var tasks = logs.Select(log => Task.Run(() => _sender.Information(log.ToString()))); - return Task.WhenAll(tasks); + foreach (LogEntry logEntry in logs) + { + _sender.Information(logEntry.ToString()); + } } #endregion public diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index 8a5fb611..d2219018 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -23,7 +23,6 @@ using Serilog.Sinks.Http; using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -59,10 +58,12 @@ namespace LogExporter.Strategy #region public - public Task ExportAsync(List logs) + public void Export(List logs) { - var tasks = logs.Select(log => Task.Run(() => _sender.Information(log.ToString()))); - return Task.WhenAll(tasks); + foreach (LogEntry logEntry in logs) + { + _sender.Information(logEntry.ToString()); + } } #endregion public diff --git a/Apps/LogExporterApp/Strategy/IExportStrategy.cs b/Apps/LogExporterApp/Strategy/IExportStrategy.cs index 63feef3b..6c1647fa 100644 --- a/Apps/LogExporterApp/Strategy/IExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/IExportStrategy.cs @@ -28,6 +28,6 @@ namespace LogExporter.Strategy /// public interface IExportStrategy: IDisposable { - Task ExportAsync(List logs); + void Export(List logs); } } diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index eee2d356..dc4bf2af 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -24,7 +24,6 @@ using Serilog.Sinks.Syslog; using System; using System.Collections.Generic; using System.Linq; -using System.Threading.Tasks; namespace LogExporter.Strategy { @@ -75,14 +74,13 @@ namespace LogExporter.Strategy #region public - public Task ExportAsync(List logs) + public void Export(List logs) { - var tasks = logs - .Select(log => Task.Run(() => - Log.Information(_formatter.FormatMessage(Convert(log)))) - ); - return Task.WhenAll(tasks); + foreach (var log in logs) + { + Log.Information(_formatter.FormatMessage(Convert(log))); + } } #endregion public From 3373034cdb7f5ac46918b4522671b7516f8c1274 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 21 Nov 2024 20:35:54 +0200 Subject: [PATCH 15/17] Fixed newline issue --- Apps/LogExporterApp/Strategy/FileExportStrategy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 181bd1c8..77dd3118 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -36,7 +36,7 @@ namespace LogExporter.Strategy public FileExportStrategy(string filePath) { - _sender = new LoggerConfiguration().WriteTo.File(filePath, outputTemplate: "{Message:lj}{Newline}").CreateLogger(); + _sender = new LoggerConfiguration().WriteTo.File(filePath, outputTemplate: "{Message:lj}{NewLine}{Exception}").CreateLogger(); } #endregion constructor From afca6ba5934292dfc315a3505963b398d3da1c42 Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Thu, 21 Nov 2024 23:14:22 +0200 Subject: [PATCH 16/17] Removed locally added DLL --- Apps/LogExporterApp/LogExporterApp.csproj | 3 --- 1 file changed, 3 deletions(-) diff --git a/Apps/LogExporterApp/LogExporterApp.csproj b/Apps/LogExporterApp/LogExporterApp.csproj index 87ffa0d9..bf64135b 100644 --- a/Apps/LogExporterApp/LogExporterApp.csproj +++ b/Apps/LogExporterApp/LogExporterApp.csproj @@ -34,9 +34,6 @@ - - Serilog.Sinks.SyslogMessages.dll - ..\..\..\TechnitiumLibrary\bin\TechnitiumLibrary.Net.dll false From 693437603b83df5d692cda55353635b3b9dae35c Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Sat, 23 Nov 2024 12:39:46 +0200 Subject: [PATCH 17/17] Reimplemented async --- Apps/LogExporterApp/Strategy/ExportManager.cs | 4 ++-- Apps/LogExporterApp/Strategy/FileExportStrategy.cs | 8 ++++++-- Apps/LogExporterApp/Strategy/HttpExportStrategy.cs | 11 +++++++---- Apps/LogExporterApp/Strategy/IExportStrategy.cs | 2 +- .../Strategy/SyslogExportStrategy.cs | 14 ++++++++------ 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Apps/LogExporterApp/Strategy/ExportManager.cs b/Apps/LogExporterApp/Strategy/ExportManager.cs index f6e2d59c..5a99dd1e 100644 --- a/Apps/LogExporterApp/Strategy/ExportManager.cs +++ b/Apps/LogExporterApp/Strategy/ExportManager.cs @@ -52,11 +52,11 @@ namespace LogExporter.Strategy return _exportStrategies.Count > 0; } - public void ImplementStrategy(List logs) + public async Task ImplementStrategy(List logs) { foreach (var strategy in _exportStrategies.Values) { - strategy.Export(logs); + await strategy.ExportAsync(logs).ConfigureAwait(false); } } diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 77dd3118..d3c90fef 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -19,6 +19,7 @@ along with this program. If not, see . using Serilog; using System.Collections.Generic; +using System.Threading.Tasks; namespace LogExporter.Strategy { @@ -43,12 +44,15 @@ namespace LogExporter.Strategy #region public - public void Export(List logs) + public Task ExportAsync(List logs) { - foreach (LogEntry logEntry in logs) + return Task.Run(() => + { + foreach (LogEntry logEntry in logs) { _sender.Information(logEntry.ToString()); } + }); } #endregion public diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index d2219018..51461d1c 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -58,12 +58,15 @@ namespace LogExporter.Strategy #region public - public void Export(List logs) + public Task ExportAsync(List logs) { - foreach (LogEntry logEntry in logs) + return Task.Run(() => { - _sender.Information(logEntry.ToString()); - } + foreach (LogEntry logEntry in logs) + { + _sender.Information(logEntry.ToString()); + } + }); } #endregion public diff --git a/Apps/LogExporterApp/Strategy/IExportStrategy.cs b/Apps/LogExporterApp/Strategy/IExportStrategy.cs index 6c1647fa..63feef3b 100644 --- a/Apps/LogExporterApp/Strategy/IExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/IExportStrategy.cs @@ -28,6 +28,6 @@ namespace LogExporter.Strategy /// public interface IExportStrategy: IDisposable { - void Export(List logs); + Task ExportAsync(List logs); } } diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index dc4bf2af..c3c5fb28 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -24,6 +24,7 @@ using Serilog.Sinks.Syslog; using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; namespace LogExporter.Strategy { @@ -74,15 +75,16 @@ namespace LogExporter.Strategy #region public - public void Export(List logs) + public Task ExportAsync(List logs) { - - foreach (var log in logs) + return Task.Run(() => { - Log.Information(_formatter.FormatMessage(Convert(log))); - } + foreach (var log in logs) + { + _sender.Information((string?)_formatter.FormatMessage((LogEvent?)Convert(log))); + } + }); } - #endregion public #region IDisposable