From 54952f19778fa6830a7f37fa5c6d19566faa284f Mon Sep 17 00:00:00 2001 From: Zafer Balkan Date: Sun, 20 Oct 2024 17:44:46 +0300 Subject: [PATCH] Simplified JSON serialization. While there was lower memory allocation, the speed was an issue. --- Apps/LogExporterApp/GrowableBuffer.cs | 132 ------------------ Apps/LogExporterApp/LogEntry.cs | 94 ++++--------- .../Strategy/FileExportStrategy.cs | 10 +- .../Strategy/HttpExportStrategy.cs | 7 +- .../Strategy/SyslogExportStrategy.cs | 4 +- 5 files changed, 40 insertions(+), 207 deletions(-) delete mode 100644 Apps/LogExporterApp/GrowableBuffer.cs diff --git a/Apps/LogExporterApp/GrowableBuffer.cs b/Apps/LogExporterApp/GrowableBuffer.cs deleted file mode 100644 index 71504ff4..00000000 --- a/Apps/LogExporterApp/GrowableBuffer.cs +++ /dev/null @@ -1,132 +0,0 @@ -using System; -using System.Buffers; - -namespace LogExporter -{ - public class GrowableBuffer : IBufferWriter, IDisposable - { - // Gets the current length of the buffer contents - public int Length => _position; - - // Initial capacity to be used in the constructor - private const int DefaultInitialCapacity = 256; - - private Memory _buffer; - - private int _position; - - private bool disposedValue; - - public GrowableBuffer(int initialCapacity = DefaultInitialCapacity) - { - _buffer = new Memory(ArrayPool.Shared.Rent(initialCapacity)); - _position = 0; - } - - // IBufferWriter implementation - public void Advance(int count) - { - if (count < 0 || _position + count > _buffer.Length) - throw new ArgumentOutOfRangeException(nameof(count)); - - _position += count; - } - - // Appends a single element to the buffer - public void Append(T item) - { - EnsureCapacity(1); - _buffer.Span[_position++] = item; - } - - // Appends a span of elements to the buffer - public void Append(ReadOnlySpan span) - { - EnsureCapacity(span.Length); - span.CopyTo(_buffer.Span[_position..]); - _position += span.Length; - } - - // Clears the buffer for reuse without reallocating - public void Clear() => _position = 0; - - public Memory GetMemory(int sizeHint = 0) - { - EnsureCapacity(sizeHint); - return _buffer[_position..]; - } - - public Span GetSpan(int sizeHint = 0) - { - EnsureCapacity(sizeHint); - return _buffer.Span[_position..]; - } - - // Returns the buffer contents as an array - public T[] ToArray() - { - T[] result = new T[_position]; - _buffer.Span[.._position].CopyTo(result); - return result; - } - - // Returns the buffer contents as a ReadOnlySpan - public ReadOnlySpan ToSpan() => _buffer.Span[.._position]; - - public override string ToString() => _buffer.Span[.._position].ToString(); - - // Ensures the buffer has enough capacity to add more elements - private void EnsureCapacity(int additionalCapacity) - { - if (_position + additionalCapacity > _buffer.Length) - { - GrowBuffer(_position + additionalCapacity); - } - } - - // Grows the buffer to accommodate the required capacity - private void GrowBuffer(int requiredCapacity) - { - int newCapacity = Math.Max(_buffer.Length * 2, requiredCapacity); - - // Rent a larger buffer from the pool - T[] newArray = ArrayPool.Shared.Rent(newCapacity); - Memory newBuffer = new Memory(newArray); - - // Copy current contents to the new buffer - _buffer.Span[.._position].CopyTo(newBuffer.Span); - - // Return old buffer to the pool - ArrayPool.Shared.Return(_buffer.ToArray()); - - // Assign the new buffer - _buffer = newBuffer; - } - - #region IDisposable - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - ArrayPool.Shared.Return(_buffer.ToArray()); - _buffer = Memory.Empty; - _position = 0; - } - } - - disposedValue = true; - } - - #endregion IDisposable - } -} \ No newline at end of file diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs index 8d185831..c7d1f9f4 100644 --- a/Apps/LogExporterApp/LogEntry.cs +++ b/Apps/LogExporterApp/LogEntry.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using TechnitiumLibrary.Net.Dns; using TechnitiumLibrary.Net.Dns.ResourceRecords; @@ -91,73 +92,38 @@ namespace LogExporter public DnssecStatus DnssecStatus { get; set; } } - public ReadOnlySpan AsSpan() + public override string ToString() { - // Initialize a ValueStringBuilder with some initial capacity - var buffer = new GrowableBuffer(256); - - using var writer = new Utf8JsonWriter(buffer); - - // Manually serialize the LogEntry as JSON - writer.WriteStartObject(); - - writer.WriteString("timestamp", Timestamp.ToUniversalTime().ToString("O")); - writer.WriteString("clientIp", ClientIp); - writer.WriteNumber("clientPort", ClientPort); - writer.WriteBoolean("dnssecOk", DnssecOk); - writer.WriteString("protocol", Protocol.ToString()); - writer.WriteString("responseCode", ResponseCode.ToString()); - - // Write Questions array - writer.WriteStartArray("questions"); - foreach (var question in Questions) - { - writer.WriteStartObject(); - writer.WriteString("questionName", question.QuestionName); - writer.WriteString("questionType", question.QuestionType.ToString()); - writer.WriteString("questionClass", question.QuestionClass.ToString()); - writer.WriteNumber("size", question.Size); - writer.WriteEndObject(); - } - writer.WriteEndArray(); - - // Write Answers array (if exists) - if (Answers != null && Answers.Count > 0) - { - writer.WriteStartArray("answers"); - foreach (var answer in Answers) - { - writer.WriteStartObject(); - writer.WriteString("recordType", answer.RecordType.ToString()); - writer.WriteString("recordData", answer.RecordData); - writer.WriteString("recordClass", answer.RecordClass.ToString()); - writer.WriteNumber("recordTtl", answer.RecordTtl); - writer.WriteNumber("size", answer.Size); - writer.WriteString("dnssecStatus", answer.DnssecStatus.ToString()); - writer.WriteEndObject(); - } - writer.WriteEndArray(); - } - - writer.WriteEndObject(); - writer.Flush(); - - return ConvertBytesToChars(buffer.ToSpan()); + return JsonSerializer.Serialize(this, DnsLogSerializerOptions.Default); } - public static Span ConvertBytesToChars(ReadOnlySpan byteSpan) + // Custom DateTime converter to handle UTC serialization in ISO 8601 format + public class JsonDateTimeConverter : JsonConverter { - // Calculate the maximum required length for the char array - int maxCharCount = Encoding.UTF8.GetCharCount(byteSpan); + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dts = reader.GetString(); + return dts == null ? DateTime.MinValue : DateTime.Parse(dts); + } - // Allocate a char array large enough to hold the converted characters - char[] charArray = new char[maxCharCount]; - - // Decode the byteSpan into the char array - int actualCharCount = Encoding.UTF8.GetChars(byteSpan, charArray); - - // Return a span of only the relevant portion of the char array - return new Span(charArray, 0, actualCharCount); + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + writer.WriteStringValue(value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ")); + } } - }; -} + + // Setup reusable options with a single instance + public static class DnsLogSerializerOptions + { + public static readonly JsonSerializerOptions Default = new JsonSerializerOptions + { + WriteIndented = false, // Newline delimited logs should not be multiline + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // Convert properties to camelCase + Converters = { new JsonStringEnumConverter(), new JsonDateTimeConverter() }, // Handle enums and DateTime conversion + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, // For safe encoding + NumberHandling = JsonNumberHandling.Strict, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull // Ignore null values + }; + } + } +} \ No newline at end of file diff --git a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs index 605012fa..351e0a13 100644 --- a/Apps/LogExporterApp/Strategy/FileExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/FileExportStrategy.cs @@ -21,6 +21,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -54,17 +55,16 @@ namespace LogExporter.Strategy public Task ExportAsync(List logs) { - var buffer = new GrowableBuffer(); + var jsonLogs = new StringBuilder(logs.Count); foreach (var log in logs) { - buffer.Append(log.AsSpan()); - buffer.Append('\n'); + jsonLogs.AppendLine(log.ToString()); } - Flush(buffer.ToSpan()); + Flush(jsonLogs.ToString()); return Task.CompletedTask; } - private void Flush(ReadOnlySpan jsonLogs) + private void Flush(string jsonLogs) { // Wait to enter the semaphore _fileSemaphore.Wait(); diff --git a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs index 3967f53a..46722017 100644 --- a/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/HttpExportStrategy.cs @@ -57,13 +57,12 @@ namespace LogExporter.Strategy public async Task ExportAsync(List logs) { - var buffer = new GrowableBuffer(); + var jsonLogs = new StringBuilder(logs.Count); foreach (var log in logs) { - buffer.Append(log.AsSpan()); - buffer.Append('\n'); + jsonLogs.AppendLine(log.ToString()); } - var content = buffer.ToString() ?? string.Empty; + var content = jsonLogs.ToString() ?? string.Empty; var request = new HttpRequestMessage { RequestUri = new Uri(_endpoint), diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index a083392c..a01c81c6 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -129,7 +129,7 @@ namespace LogExporter.Strategy }); // Add each question to the structured data - if (log.Questions != null && log.Questions.Count > 0) + if (log.Questions?.Count > 0) { for (int i = 0; i < log.Questions.Count; i++) { @@ -142,7 +142,7 @@ namespace LogExporter.Strategy } // Add each answer to the structured data - if (log.Answers != null && log.Answers.Count > 0) + if (log.Answers?.Count > 0) { for (int i = 0; i < log.Answers.Count; i++) {