Simplified JSON serialization. While there was lower memory allocation, the speed was an issue.

This commit is contained in:
Zafer Balkan
2024-10-20 17:44:46 +03:00
parent fb42498485
commit 54952f1977
5 changed files with 40 additions and 207 deletions

View File

@@ -1,132 +0,0 @@
using System;
using System.Buffers;
namespace LogExporter
{
public class GrowableBuffer<T> : IBufferWriter<T>, IDisposable
{
// Gets the current length of the buffer contents
public int Length => _position;
// Initial capacity to be used in the constructor
private const int DefaultInitialCapacity = 256;
private Memory<T> _buffer;
private int _position;
private bool disposedValue;
public GrowableBuffer(int initialCapacity = DefaultInitialCapacity)
{
_buffer = new Memory<T>(ArrayPool<T>.Shared.Rent(initialCapacity));
_position = 0;
}
// IBufferWriter<T> implementation
public void Advance(int count)
{
if (count < 0 || _position + count > _buffer.Length)
throw new ArgumentOutOfRangeException(nameof(count));
_position += count;
}
// Appends a single element to the buffer
public void Append(T item)
{
EnsureCapacity(1);
_buffer.Span[_position++] = item;
}
// Appends a span of elements to the buffer
public void Append(ReadOnlySpan<T> span)
{
EnsureCapacity(span.Length);
span.CopyTo(_buffer.Span[_position..]);
_position += span.Length;
}
// Clears the buffer for reuse without reallocating
public void Clear() => _position = 0;
public Memory<T> GetMemory(int sizeHint = 0)
{
EnsureCapacity(sizeHint);
return _buffer[_position..];
}
public Span<T> GetSpan(int sizeHint = 0)
{
EnsureCapacity(sizeHint);
return _buffer.Span[_position..];
}
// Returns the buffer contents as an array
public T[] ToArray()
{
T[] result = new T[_position];
_buffer.Span[.._position].CopyTo(result);
return result;
}
// Returns the buffer contents as a ReadOnlySpan<T>
public ReadOnlySpan<T> ToSpan() => _buffer.Span[.._position];
public override string ToString() => _buffer.Span[.._position].ToString();
// Ensures the buffer has enough capacity to add more elements
private void EnsureCapacity(int additionalCapacity)
{
if (_position + additionalCapacity > _buffer.Length)
{
GrowBuffer(_position + additionalCapacity);
}
}
// Grows the buffer to accommodate the required capacity
private void GrowBuffer(int requiredCapacity)
{
int newCapacity = Math.Max(_buffer.Length * 2, requiredCapacity);
// Rent a larger buffer from the pool
T[] newArray = ArrayPool<T>.Shared.Rent(newCapacity);
Memory<T> newBuffer = new Memory<T>(newArray);
// Copy current contents to the new buffer
_buffer.Span[.._position].CopyTo(newBuffer.Span);
// Return old buffer to the pool
ArrayPool<T>.Shared.Return(_buffer.ToArray());
// Assign the new buffer
_buffer = newBuffer;
}
#region IDisposable
public void Dispose()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
ArrayPool<T>.Shared.Return(_buffer.ToArray());
_buffer = Memory<T>.Empty;
_position = 0;
}
}
disposedValue = true;
}
#endregion IDisposable
}
}

View File

@@ -4,6 +4,7 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using TechnitiumLibrary.Net.Dns;
using TechnitiumLibrary.Net.Dns.ResourceRecords;
@@ -91,73 +92,38 @@ namespace LogExporter
public DnssecStatus DnssecStatus { get; set; }
}
public ReadOnlySpan<char> AsSpan()
public override string ToString()
{
// Initialize a ValueStringBuilder with some initial capacity
var buffer = new GrowableBuffer<byte>(256);
using var writer = new Utf8JsonWriter(buffer);
// Manually serialize the LogEntry as JSON
writer.WriteStartObject();
writer.WriteString("timestamp", Timestamp.ToUniversalTime().ToString("O"));
writer.WriteString("clientIp", ClientIp);
writer.WriteNumber("clientPort", ClientPort);
writer.WriteBoolean("dnssecOk", DnssecOk);
writer.WriteString("protocol", Protocol.ToString());
writer.WriteString("responseCode", ResponseCode.ToString());
// Write Questions array
writer.WriteStartArray("questions");
foreach (var question in Questions)
{
writer.WriteStartObject();
writer.WriteString("questionName", question.QuestionName);
writer.WriteString("questionType", question.QuestionType.ToString());
writer.WriteString("questionClass", question.QuestionClass.ToString());
writer.WriteNumber("size", question.Size);
writer.WriteEndObject();
}
writer.WriteEndArray();
// Write Answers array (if exists)
if (Answers != null && Answers.Count > 0)
{
writer.WriteStartArray("answers");
foreach (var answer in Answers)
{
writer.WriteStartObject();
writer.WriteString("recordType", answer.RecordType.ToString());
writer.WriteString("recordData", answer.RecordData);
writer.WriteString("recordClass", answer.RecordClass.ToString());
writer.WriteNumber("recordTtl", answer.RecordTtl);
writer.WriteNumber("size", answer.Size);
writer.WriteString("dnssecStatus", answer.DnssecStatus.ToString());
writer.WriteEndObject();
}
writer.WriteEndArray();
}
writer.WriteEndObject();
writer.Flush();
return ConvertBytesToChars(buffer.ToSpan());
return JsonSerializer.Serialize(this, DnsLogSerializerOptions.Default);
}
public static Span<char> ConvertBytesToChars(ReadOnlySpan<byte> byteSpan)
// Custom DateTime converter to handle UTC serialization in ISO 8601 format
public class JsonDateTimeConverter : JsonConverter<DateTime>
{
// Calculate the maximum required length for the char array
int maxCharCount = Encoding.UTF8.GetCharCount(byteSpan);
public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var dts = reader.GetString();
return dts == null ? DateTime.MinValue : DateTime.Parse(dts);
}
// Allocate a char array large enough to hold the converted characters
char[] charArray = new char[maxCharCount];
// Decode the byteSpan into the char array
int actualCharCount = Encoding.UTF8.GetChars(byteSpan, charArray);
// Return a span of only the relevant portion of the char array
return new Span<char>(charArray, 0, actualCharCount);
public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options)
{
writer.WriteStringValue(value.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ"));
}
}
};
}
// Setup reusable options with a single instance
public static class DnsLogSerializerOptions
{
public static readonly JsonSerializerOptions Default = new JsonSerializerOptions
{
WriteIndented = false, // Newline delimited logs should not be multiline
PropertyNamingPolicy = JsonNamingPolicy.CamelCase, // Convert properties to camelCase
Converters = { new JsonStringEnumConverter(), new JsonDateTimeConverter() }, // Handle enums and DateTime conversion
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, // For safe encoding
NumberHandling = JsonNumberHandling.Strict,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull // Ignore null values
};
}
}
}

View File

@@ -21,6 +21,7 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -54,17 +55,16 @@ namespace LogExporter.Strategy
public Task ExportAsync(List<LogEntry> logs)
{
var buffer = new GrowableBuffer<char>();
var jsonLogs = new StringBuilder(logs.Count);
foreach (var log in logs)
{
buffer.Append(log.AsSpan());
buffer.Append('\n');
jsonLogs.AppendLine(log.ToString());
}
Flush(buffer.ToSpan());
Flush(jsonLogs.ToString());
return Task.CompletedTask;
}
private void Flush(ReadOnlySpan<char> jsonLogs)
private void Flush(string jsonLogs)
{
// Wait to enter the semaphore
_fileSemaphore.Wait();

View File

@@ -57,13 +57,12 @@ namespace LogExporter.Strategy
public async Task ExportAsync(List<LogEntry> logs)
{
var buffer = new GrowableBuffer<char>();
var jsonLogs = new StringBuilder(logs.Count);
foreach (var log in logs)
{
buffer.Append(log.AsSpan());
buffer.Append('\n');
jsonLogs.AppendLine(log.ToString());
}
var content = buffer.ToString() ?? string.Empty;
var content = jsonLogs.ToString() ?? string.Empty;
var request = new HttpRequestMessage
{
RequestUri = new Uri(_endpoint),

View File

@@ -129,7 +129,7 @@ namespace LogExporter.Strategy
});
// Add each question to the structured data
if (log.Questions != null && log.Questions.Count > 0)
if (log.Questions?.Count > 0)
{
for (int i = 0; i < log.Questions.Count; i++)
{
@@ -142,7 +142,7 @@ namespace LogExporter.Strategy
}
// Add each answer to the structured data
if (log.Answers != null && log.Answers.Count > 0)
if (log.Answers?.Count > 0)
{
for (int i = 0; i < log.Answers.Count; i++)
{