fergalmoran/DnsServer
1
0
Fork 0
mirror of https://github.com/fergalmoran/DnsServer.git synced 2025-12-22 09:29:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

FailoverApp: implemented under maintanence feature to indicate if an address is taken down for maintenance. Code refactoring done.

Browse Source
This commit is contained in:
Shreyas Zare
2021-09-01 18:00:09 +05:30
parent 9953db90c0
commit 8d85c9bbae
10 changed files with 333 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Apps/FailoverApp/Address.cs
View File

@@ -78,11 +78,17 @@ namespace Failover
if (address.AddressFamily == AddressFamily.InterNetwork)
{
HealthCheckStatus status = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true);
if (status is null)
HealthCheckResponse response = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true);
switch (response.Status)
{
case HealthStatus.Unknown:
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.A, question.Class, 30, new DnsARecord(address)));
else if (status.IsHealthy)
break;
case HealthStatus.Healthy:
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.A, question.Class, appRecordTtl, new DnsARecord(address)));
break;
}
}
}
break;
@@ -94,11 +100,17 @@ namespace Failover
if (address.AddressFamily == AddressFamily.InterNetworkV6)
{
HealthCheckStatus status = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true);
if (status is null)
HealthCheckResponse response = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true);
switch (response.Status)
{
case HealthStatus.Unknown:
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.AAAA, question.Class, 30, new DnsAAAARecord(address)));
else if (status.IsHealthy)
break;
case HealthStatus.Healthy:
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.AAAA, question.Class, appRecordTtl, new DnsAAAARecord(address)));
break;
}
}
}
break;
@@ -113,16 +125,12 @@ namespace Failover
foreach (dynamic jsonAddress in jsonAddresses)
{
IPAddress address = IPAddress.Parse(jsonAddress.Value);
HealthCheckStatus status = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, false);
HealthCheckResponse response = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, false);
string text = "app=failover; addressType=" + type.ToString() + "; address=" + address.ToString() + "; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri);
string text = "app=failover; addressType=" + type.ToString() + "; address=" + address.ToString() + "; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri) + "; healthStatus=" + response.Status.ToString() + ";";
if (status is null)
text += "; healthStatus=Unknown;";
else if (status.IsHealthy)
text += "; healthStatus=Healthy;";
else
text += "; healthStatus=Failed; failureReason=" + status.FailureReason + ";";
if (response.Status == HealthStatus.Failed)
text += " failureReason=" + response.FailureReason + ";";
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, appRecordTtl, new DnsTXTRecord(text)));
}

34
Apps/FailoverApp/CNAME.cs
View File

@@ -57,16 +57,16 @@ namespace Failover
private IReadOnlyList<DnsResourceRecord> GetAnswers(string domain, DnsQuestionRecord question, string zoneName, uint appRecordTtl, string healthCheck, Uri healthCheckUrl)
{
HealthCheckStatus status = _healthService.QueryStatus(domain, question.Type, healthCheck, healthCheckUrl, true);
if (status is null)
HealthCheckResponse response = _healthService.QueryStatus(domain, question.Type, healthCheck, healthCheckUrl, true);
switch (response.Status)
{
case HealthStatus.Unknown:
if (question.Name.Equals(zoneName, StringComparison.OrdinalIgnoreCase)) //check for zone apex
return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.ANAME, DnsClass.IN, 30, new DnsANAMERecord(domain)) }; //use ANAME
else
return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.CNAME, DnsClass.IN, 30, new DnsCNAMERecord(domain)) };
}
else if (status.IsHealthy)
{
case HealthStatus.Healthy:
if (question.Name.Equals(zoneName, StringComparison.OrdinalIgnoreCase)) //check for zone apex
return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.ANAME, DnsClass.IN, appRecordTtl, new DnsANAMERecord(domain)) }; //use ANAME
else
@@ -79,31 +79,23 @@ namespace Failover
private void GetStatusAnswers(string domain, FailoverType type, DnsQuestionRecord question, uint appRecordTtl, string healthCheck, Uri healthCheckUrl, List<DnsResourceRecord> answers)
{
{
HealthCheckStatus status = _healthService.QueryStatus(domain, DnsResourceRecordType.A, healthCheck, healthCheckUrl, false);
HealthCheckResponse response = _healthService.QueryStatus(domain, DnsResourceRecordType.A, healthCheck, healthCheckUrl, false);
string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: A; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri);
string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: A; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri) + "; healthStatus=" + response.Status.ToString() + ";";
if (status is null)
text += "; healthStatus=Unknown;";
else if (status.IsHealthy)
text += "; healthStatus=Healthy;";
else
text += "; healthStatus=Failed; failureReason=" + status.FailureReason + ";";
if (response.Status == HealthStatus.Failed)
text += " failureReason=" + response.FailureReason + ";";
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, appRecordTtl, new DnsTXTRecord(text)));
}
{
HealthCheckStatus status = _healthService.QueryStatus(domain, DnsResourceRecordType.AAAA, healthCheck, healthCheckUrl, false);
HealthCheckResponse response = _healthService.QueryStatus(domain, DnsResourceRecordType.AAAA, healthCheck, healthCheckUrl, false);
string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: AAAA; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri);
string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: AAAA; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri) + "; healthStatus=" + response.Status.ToString() + ";";
if (status is null)
text += "; healthStatus=Unknown;";
else if (status.IsHealthy)
text += "; healthStatus=Healthy;";
else
text += "; healthStatus=Failed; failureReason=" + status.FailureReason + ";";
if (response.Status == HealthStatus.Failed)
text += " failureReason=" + response.FailureReason + ";";
answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, appRecordTtl, new DnsTXTRecord(text)));
}

112
Apps/FailoverApp/EmailAlert.cs
View File

@@ -188,7 +188,7 @@ namespace Failover
_smtpClient.Proxy = _service.DnsServer.Proxy;
}
public Task SendAlertAsync(IPAddress address, string healthCheck, HealthCheckStatus healthCheckStatus)
public Task SendAlertAsync(IPAddress address, string healthCheck, HealthCheckResponse healthCheckResponse)
{
if (!_enabled || (_mailFrom is null) || (_alertTo is null) || (_alertTo.Length == 0))
return Task.CompletedTask;
@@ -200,32 +200,40 @@ namespace Failover
foreach (MailAddress alertTo in _alertTo)
message.To.Add(alertTo);
if (healthCheckStatus.IsHealthy)
message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is " + healthCheckResponse.Status.ToString().ToUpper();
switch (healthCheckResponse.Status)
{
message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is Healthy";
message.Body = @"Hi,
The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"] and found that the address was healthy.
Alert time: " + healthCheckStatus.DateTime.ToString("R") + @"
Regards,
DNS Failover App
";
}
else
{
message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is Failed";
case HealthStatus.Failed:
message.Body = @"Hi,
The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"] and found that the address failed to respond.
The failure reason is: " + healthCheckStatus.FailureReason + @"
Alert time: " + healthCheckStatus.DateTime.ToString("R") + @"
Address: " + address.ToString() + @"
Health Check: " + healthCheck + @"
Status: " + healthCheckResponse.Status.ToString().ToUpper() + @"
Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @"
Failure Reason: " + healthCheckResponse.FailureReason + @"
Regards,
DNS Failover App
";
break;
default:
message.Body = @"Hi,
The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"] and found that the address status was " + healthCheckResponse.Status.ToString().ToUpper() + @".
Address: " + address.ToString() + @"
Health Check: " + healthCheck + @"
Status: " + healthCheckResponse.Status.ToString().ToUpper() + @"
Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @"
Regards,
DNS Failover App
";
break;
}
return SendMailAsync(message);
@@ -243,13 +251,16 @@ DNS Failover App
foreach (MailAddress alertTo in _alertTo)
message.To.Add(alertTo);
message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is Error";
message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is ERROR";
message.Body = @"Hi,
The DNS Failover App has failed to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"].
The error description is: " + ex.ToString() + @"
Alert time: " + DateTime.UtcNow.ToString("R") + @"
Address: " + address.ToString() + @"
Health Check: " + healthCheck + @"
Status: ERROR
Alert Time: " + DateTime.UtcNow.ToString("R") + @"
Failure Reason: " + ex.ToString() + @"
Regards,
DNS Failover App
@@ -258,7 +269,7 @@ DNS Failover App
return SendMailAsync(message);
}
public Task SendAlertAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckStatus healthCheckStatus)
public Task SendAlertAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckResponse healthCheckResponse)
{
if (!_enabled || (_mailFrom is null) || (_alertTo is null) || (_alertTo.Length == 0))
return Task.CompletedTask;
@@ -270,34 +281,42 @@ DNS Failover App
foreach (MailAddress alertTo in _alertTo)
message.To.Add(alertTo);
if (healthCheckStatus.IsHealthy)
message.Subject = "[Alert] Domain [" + domain + "] Status Is " + healthCheckResponse.Status.ToString().ToUpper();
switch(healthCheckResponse.Status)
{
message.Subject = "[Alert] Domain [" + domain + "] Status Is Healthy";
message.Body = @"Hi,
The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"] and found that the domain name was healthy.
DNS record type: " + type.ToString() + @"
Alert time: " + healthCheckStatus.DateTime.ToString("R") + @"
Regards,
DNS Failover App
";
}
else
{
message.Subject = "[Alert] Domain [" + domain + "] Status Is Failed";
case HealthStatus.Failed:
message.Body = @"Hi,
The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"] and found that the domain name failed to respond.
The failure reason is: " + healthCheckStatus.FailureReason + @"
DNS record type: " + type.ToString() + @"
Alert time: " + healthCheckStatus.DateTime.ToString("R") + @"
Domain: " + domain + @"
Record Type: " + type.ToString() + @"
Health Check: " + healthCheck + @"
Status: " + healthCheckResponse.Status.ToString().ToUpper() + @"
Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @"
Failure Reason: " + healthCheckResponse.FailureReason + @"
Regards,
DNS Failover App
";
break;
default:
message.Body = @"Hi,
The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"] and found that the domain name status was " + healthCheckResponse.Status.ToString().ToUpper() + @".
Domain: " + domain + @"
Record Type: " + type.ToString() + @"
Health Check: " + healthCheck + @"
Status: " + healthCheckResponse.Status.ToString().ToUpper() + @"
Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @"
Regards,
DNS Failover App
";
break;
}
return SendMailAsync(message);
@@ -315,14 +334,17 @@ DNS Failover App
foreach (MailAddress alertTo in _alertTo)
message.To.Add(alertTo);
message.Subject = "[Alert] Domain [" + domain + "] Status Is Error";
message.Subject = "[Alert] Domain [" + domain + "] Status Is ERROR";
message.Body = @"Hi,
The DNS Failover App has failed to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"].
The error description is: " + ex.ToString() + @"
DNS record type: " + type.ToString() + @"
Alert time: " + DateTime.UtcNow.ToString("R") + @"
Domain: " + domain + @"
Record Type: " + type.ToString() + @"
Health Check: " + healthCheck + @"
Status: ERROR
Alert Time: " + DateTime.UtcNow.ToString("R") + @"
Failure Reason: " + ex.ToString() + @"
Regards,
DNS Failover App

2
Apps/FailoverApp/FailoverApp.csproj
View File

@@ -4,7 +4,7 @@
<TargetFramework>net5.0</TargetFramework>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
<Version>1.4</Version>
<Version>1.6</Version>
<Company>Technitium</Company>
<Product>Technitium DNS Server</Product>
<Authors>Shreyas Zare</Authors>

62
Apps/FailoverApp/HealthCheck.cs
View File

@@ -25,6 +25,7 @@ using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Threading.Tasks;
using TechnitiumLibrary.IO;
using TechnitiumLibrary.Net;
using TechnitiumLibrary.Net.Dns;
using TechnitiumLibrary.Net.Proxy;
@@ -258,7 +259,7 @@ namespace Failover
ConditionalHttpReload();
}
public async Task<HealthCheckStatus> IsHealthyAsync(string domain, DnsResourceRecordType type, Uri healthCheckUrl)
public async Task<HealthCheckResponse> IsHealthyAsync(string domain, DnsResourceRecordType type, Uri healthCheckUrl)
{
switch (type)
{
@@ -266,57 +267,68 @@ namespace Failover
{
DnsDatagram response = await _service.DnsServer.DirectQueryAsync(new DnsQuestionRecord(domain, type, DnsClass.IN));
if ((response is null) || (response.Answer.Count == 0))
return new HealthCheckStatus(false, "Failed to resolve address.", null);
return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address.");
IReadOnlyList<IPAddress> addresses = DnsClient.ParseResponseA(response);
if (addresses.Count > 0)
{
HealthCheckStatus lastStatus = null;
HealthCheckResponse lastResponse = null;
foreach (IPAddress address in addresses)
{
lastStatus = await IsHealthyAsync(address, healthCheckUrl);
if (lastStatus.IsHealthy)
return lastStatus;
lastResponse = await IsHealthyAsync(address, healthCheckUrl);
if (lastResponse.Status == HealthStatus.Healthy)
return lastResponse;
}
return lastStatus;
return lastResponse;
}
return new HealthCheckStatus(false, "Failed to resolve address.", null);
return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address.");
}
case DnsResourceRecordType.AAAA:
{
DnsDatagram response = await _service.DnsServer.DirectQueryAsync(new DnsQuestionRecord(domain, type, DnsClass.IN));
if ((response is null) || (response.Answer.Count == 0))
return new HealthCheckStatus(false, "Failed to resolve address.", null);
return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address.");
IReadOnlyList<IPAddress> addresses = DnsClient.ParseResponseAAAA(response);
if (addresses.Count > 0)
{
HealthCheckStatus lastStatus = null;
HealthCheckResponse lastResponse = null;
foreach (IPAddress address in addresses)
{
lastStatus = await IsHealthyAsync(address, healthCheckUrl);
if (lastStatus.IsHealthy)
return lastStatus;
lastResponse = await IsHealthyAsync(address, healthCheckUrl);
if (lastResponse.Status == HealthStatus.Healthy)
return lastResponse;
}
return lastStatus;
return lastResponse;
}
return new HealthCheckStatus(false, "Failed to resolve address.", null);
return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address.");
}
default:
return new HealthCheckStatus(false, "Not supported.", null);
return new HealthCheckResponse(HealthStatus.Failed, "Not supported.");
}
}
public async Task<HealthCheckStatus> IsHealthyAsync(IPAddress address, Uri healthCheckUrl)
public async Task<HealthCheckResponse> IsHealthyAsync(IPAddress address, Uri healthCheckUrl)
{
foreach (KeyValuePair<NetworkAddress, bool> network in _service.UnderMaintenance)
{
if (network.Key.Contains(address))
{
if (network.Value)
return new HealthCheckResponse(HealthStatus.Maintenance);
break;
}
}
switch (_type)
{
case HealthCheckType.Ping:
@@ -332,13 +344,13 @@ namespace Failover
{
PingReply reply = await ping.SendPingAsync(address, _timeout);
if (reply.Status == IPStatus.Success)
return new HealthCheckStatus(true, null, null);
return new HealthCheckResponse(HealthStatus.Healthy);
lastReason = reply.Status.ToString();
}
while (++retry < _retries);
return new HealthCheckStatus(false, lastReason, null);
return new HealthCheckResponse(HealthStatus.Failed, lastReason);
}
}
@@ -368,7 +380,7 @@ namespace Failover
}
}
return new HealthCheckStatus(true, null, null);
return new HealthCheckResponse(HealthStatus.Healthy);
}
catch (TimeoutException ex)
{
@@ -387,7 +399,7 @@ namespace Failover
}
while (++retry < _retries);
return new HealthCheckStatus(false, lastReason, lastException);
return new HealthCheckResponse(HealthStatus.Failed, lastReason, lastException);
}
case HealthCheckType.Http:
@@ -410,7 +422,7 @@ namespace Failover
url = _url;
if (url is null)
return new HealthCheckStatus(false, "Missing health check URL in APP record as well as in app config.", null);
return new HealthCheckResponse(HealthStatus.Failed, "Missing health check URL in APP record as well as in app config.");
if (_type == HealthCheckType.Http)
{
@@ -434,9 +446,9 @@ namespace Failover
HttpResponseMessage httpResponse = await _httpClient.SendAsync(httpRequest);
if (httpResponse.IsSuccessStatusCode)
return new HealthCheckStatus(true, null, null);
return new HealthCheckResponse(HealthStatus.Healthy);
return new HealthCheckStatus(false, "Received HTTP status code: " + (int)httpResponse.StatusCode + " " + httpResponse.StatusCode.ToString() + "; URL: " + url.AbsoluteUri, null);
return new HealthCheckResponse(HealthStatus.Failed, "Received HTTP status code: " + (int)httpResponse.StatusCode + " " + httpResponse.StatusCode.ToString() + "; URL: " + url.AbsoluteUri);
}
catch (TaskCanceledException ex)
{
@@ -455,7 +467,7 @@ namespace Failover
}
while (++retry < _retries);
return new HealthCheckStatus(false, lastReason, lastException);
return new HealthCheckResponse(HealthStatus.Failed, lastReason, lastException);
}
default:

16
Apps/FailoverApp/HealthCheckStatus.cs → Apps/FailoverApp/HealthCheckResponse.cs
View File

@@ -21,12 +21,20 @@ using System;
namespace Failover
{
class HealthCheckStatus
enum HealthStatus
{
Unknown = 0,
Failed = 1,
Healthy = 2,
Maintenance = 3
}
class HealthCheckResponse
{
#region variables
public readonly DateTime DateTime = DateTime.UtcNow;
public readonly bool IsHealthy;
public readonly HealthStatus Status;
public readonly string FailureReason;
public readonly Exception Exception;
@@ -34,9 +42,9 @@ namespace Failover
#region constructor
public HealthCheckStatus(bool isHealthy, string failureReason, Exception exception)
public HealthCheckResponse(HealthStatus status, string failureReason = null, Exception exception = null)
{
IsHealthy = isHealthy;
Status = status;
FailureReason = failureReason;
Exception = exception;
}

146
Apps/FailoverApp/HealthMonitor.cs
View File

@@ -38,10 +38,10 @@ namespace Failover
readonly Timer _healthCheckTimer;
const int HEALTH_CHECK_TIMER_INITIAL_INTERVAL = 1000;
HealthCheckStatus _healthCheckStatus;
HealthCheckResponse _lastHealthCheckResponse;
const int MONITOR_EXPIRY = 1 * 60 * 60 * 1000; //1 hour
DateTime _lastStatusCheckedOn;
DateTime _lastHealthStatusCheckedOn;
#endregion
@@ -59,52 +59,77 @@ namespace Failover
{
if (_healthCheck is null)
{
_healthCheckStatus = null;
_lastHealthCheckResponse = null;
}
else
{
HealthCheckStatus healthCheckStatus = await _healthCheck.IsHealthyAsync(_address, healthCheckUrl);
HealthCheckResponse healthCheckResponse = await _healthCheck.IsHealthyAsync(_address, healthCheckUrl);
bool sendAlert = false;
bool statusChanged = false;
bool maintenance = false;
if (_healthCheckStatus is null)
if (_lastHealthCheckResponse is null)
{
if (!healthCheckStatus.IsHealthy)
sendAlert = true;
switch (healthCheckResponse.Status)
{
case HealthStatus.Failed:
statusChanged = true;
break;
case HealthStatus.Maintenance:
statusChanged = true;
maintenance = true;
break;
}
}
else
{
if (_healthCheckStatus.IsHealthy != healthCheckStatus.IsHealthy)
sendAlert = true;
if (_lastHealthCheckResponse.Status != healthCheckResponse.Status)
{
statusChanged = true;
if ((_lastHealthCheckResponse.Status == HealthStatus.Maintenance) || (healthCheckResponse.Status == HealthStatus.Maintenance))
maintenance = true;
}
}
if (sendAlert)
if (statusChanged)
{
if (healthCheckStatus.IsHealthy)
_dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is HEALTHY based on '" + _healthCheck.Name + "' health check.");
else
_dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckStatus.FailureReason);
switch (healthCheckResponse.Status)
{
case HealthStatus.Failed:
_dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckResponse.FailureReason);
break;
if (healthCheckStatus.Exception is not null)
_dnsServer.WriteLog(healthCheckStatus.Exception);
default:
_dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is " + healthCheckResponse.Status.ToString().ToUpper() + " based on '" + _healthCheck.Name + "' health check.");
break;
}
if (healthCheckResponse.Exception is not null)
_dnsServer.WriteLog(healthCheckResponse.Exception);
if (!maintenance)
{
//avoid sending email alerts when switching from or to maintenance
EmailAlert emailAlert = _healthCheck.EmailAlert;
if (emailAlert is not null)
_ = emailAlert.SendAlertAsync(_address, _healthCheck.Name, healthCheckStatus);
_ = emailAlert.SendAlertAsync(_address, _healthCheck.Name, healthCheckResponse);
}
WebHook webHook = _healthCheck.WebHook;
if (webHook is not null)
_ = webHook.CallAsync(_address, _healthCheck.Name, healthCheckStatus);
_ = webHook.CallAsync(_address, _healthCheck.Name, healthCheckResponse);
}
_healthCheckStatus = healthCheckStatus;
_lastHealthCheckResponse = healthCheckResponse;
}
}
catch (Exception ex)
{
_dnsServer.WriteLog(ex);
if (_healthCheckStatus is null)
if (_lastHealthCheckResponse is null)
{
EmailAlert emailAlert = _healthCheck.EmailAlert;
if (emailAlert is not null)
@@ -114,11 +139,11 @@ namespace Failover
if (webHook is not null)
_ = webHook.CallAsync(_address, _healthCheck.Name, ex);
_healthCheckStatus = new HealthCheckStatus(false, ex.ToString(), ex);
_lastHealthCheckResponse = new HealthCheckResponse(HealthStatus.Failed, ex.ToString(), ex);
}
else
{
_healthCheckStatus = null;
_lastHealthCheckResponse = null;
}
}
finally
@@ -144,52 +169,77 @@ namespace Failover
{
if (_healthCheck is null)
{
_healthCheckStatus = null;
_lastHealthCheckResponse = null;
}
else
{
HealthCheckStatus healthCheckStatus = await _healthCheck.IsHealthyAsync(_domain, _type, healthCheckUrl);
HealthCheckResponse healthCheckResponse = await _healthCheck.IsHealthyAsync(_domain, _type, healthCheckUrl);
bool sendAlert = false;
bool statusChanged = false;
bool maintenance = false;
if (_healthCheckStatus is null)
if (_lastHealthCheckResponse is null)
{
if (!healthCheckStatus.IsHealthy)
sendAlert = true;
switch (healthCheckResponse.Status)
{
case HealthStatus.Failed:
statusChanged = true;
break;
case HealthStatus.Maintenance:
statusChanged = true;
maintenance = true;
break;
}
}
else
{
if (_healthCheckStatus.IsHealthy != healthCheckStatus.IsHealthy)
sendAlert = true;
if (_lastHealthCheckResponse.Status != healthCheckResponse.Status)
{
statusChanged = true;
if ((_lastHealthCheckResponse.Status == HealthStatus.Maintenance) || (healthCheckResponse.Status == HealthStatus.Maintenance))
maintenance = true;
}
}
if (sendAlert)
if (statusChanged)
{
if (healthCheckStatus.IsHealthy)
_dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is HEALTHY based on '" + _healthCheck.Name + "' health check.");
else
_dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckStatus.FailureReason);
switch (healthCheckResponse.Status)
{
case HealthStatus.Failed:
_dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckResponse.FailureReason);
break;
if (healthCheckStatus.Exception is not null)
_dnsServer.WriteLog(healthCheckStatus.Exception);
default:
_dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is " + healthCheckResponse.Status.ToString().ToUpper() + " based on '" + _healthCheck.Name + "' health check.");
break;
}
if (healthCheckResponse.Exception is not null)
_dnsServer.WriteLog(healthCheckResponse.Exception);
if (!maintenance)
{
//avoid sending email alerts when switching from or to maintenance
EmailAlert emailAlert = _healthCheck.EmailAlert;
if (emailAlert is not null)
_ = emailAlert.SendAlertAsync(_domain, _type, _healthCheck.Name, healthCheckStatus);
_ = emailAlert.SendAlertAsync(_domain, _type, _healthCheck.Name, healthCheckResponse);
}
WebHook webHook = _healthCheck.WebHook;
if (webHook is not null)
_ = webHook.CallAsync(_domain, _type, _healthCheck.Name, healthCheckStatus);
_ = webHook.CallAsync(_domain, _type, _healthCheck.Name, healthCheckResponse);
}
_healthCheckStatus = healthCheckStatus;
_lastHealthCheckResponse = healthCheckResponse;
}
}
catch (Exception ex)
{
_dnsServer.WriteLog(ex);
if (_healthCheckStatus is null)
if (_lastHealthCheckResponse is null)
{
EmailAlert emailAlert = _healthCheck.EmailAlert;
if (emailAlert is not null)
@@ -199,11 +249,11 @@ namespace Failover
if (webHook is not null)
_ = webHook.CallAsync(_domain, _type, _healthCheck.Name, ex);
_healthCheckStatus = new HealthCheckStatus(false, ex.ToString(), ex);
_lastHealthCheckResponse = new HealthCheckResponse(HealthStatus.Failed, ex.ToString(), ex);
}
else
{
_healthCheckStatus = null;
_lastHealthCheckResponse = null;
}
}
finally
@@ -248,19 +298,19 @@ namespace Failover
public bool IsExpired()
{
return DateTime.UtcNow > _lastStatusCheckedOn.AddMilliseconds(MONITOR_EXPIRY);
return DateTime.UtcNow > _lastHealthStatusCheckedOn.AddMilliseconds(MONITOR_EXPIRY);
}
#endregion
#region properties
public HealthCheckStatus HealthCheckStatus
public HealthCheckResponse LastHealthCheckResponse
{
get
{
_lastStatusCheckedOn = DateTime.UtcNow;
return _healthCheckStatus;
_lastHealthStatusCheckedOn = DateTime.UtcNow;
return _lastHealthCheckResponse;
}
}

45
Apps/FailoverApp/HealthService.cs
View File

@@ -23,6 +23,7 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net;
using System.Threading;
using TechnitiumLibrary.Net;
using TechnitiumLibrary.Net.Dns;
namespace Failover
@@ -38,6 +39,7 @@ namespace Failover
readonly ConcurrentDictionary<string, HealthCheck> _healthChecks = new ConcurrentDictionary<string, HealthCheck>(1, 5);
readonly ConcurrentDictionary<string, EmailAlert> _emailAlerts = new ConcurrentDictionary<string, EmailAlert>(1, 2);
readonly ConcurrentDictionary<string, WebHook> _webHooks = new ConcurrentDictionary<string, WebHook>(1, 2);
readonly ConcurrentDictionary<NetworkAddress, bool> _underMaintenance = new ConcurrentDictionary<NetworkAddress, bool>();
readonly ConcurrentDictionary<string, HealthMonitor> _healthMonitors = new ConcurrentDictionary<string, HealthMonitor>();
@@ -347,14 +349,28 @@ namespace Failover
}
}
}
//under maintenance networks
_underMaintenance.Clear();
if (jsonConfig.underMaintenance is not null)
{
foreach (dynamic jsonNetwork in jsonConfig.underMaintenance)
{
string network = jsonNetwork.network.Value;
bool enable = jsonNetwork.enable.Value;
_underMaintenance.TryAdd(NetworkAddress.Parse(network), enable);
}
}
}
public HealthCheckStatus QueryStatus(IPAddress address, string healthCheck, Uri healthCheckUrl, bool tryAdd)
public HealthCheckResponse QueryStatus(IPAddress address, string healthCheck, Uri healthCheckUrl, bool tryAdd)
{
string healthMonitorKey = GetHealthMonitorKey(address, healthCheck, healthCheckUrl);
if (_healthMonitors.TryGetValue(healthMonitorKey, out HealthMonitor monitor))
return monitor.HealthCheckStatus;
return monitor.LastHealthCheckResponse;
if (_healthChecks.TryGetValue(healthCheck, out HealthCheck existingHealthCheck))
{
@@ -366,20 +382,22 @@ namespace Failover
monitor.Dispose(); //failed to add first
}
return null;
return new HealthCheckResponse(HealthStatus.Unknown);
}
else
{
return new HealthCheckResponse(HealthStatus.Failed, "No such health check: " + healthCheck);
}
}
return new HealthCheckStatus(false, "No such health check: " + healthCheck, null);
}
public HealthCheckStatus QueryStatus(string domain, DnsResourceRecordType type, string healthCheck, Uri healthCheckUrl, bool tryAdd)
public HealthCheckResponse QueryStatus(string domain, DnsResourceRecordType type, string healthCheck, Uri healthCheckUrl, bool tryAdd)
{
domain = domain.ToLower();
string healthMonitorKey = GetHealthMonitorKey(domain, type, healthCheck, healthCheckUrl);
if (_healthMonitors.TryGetValue(healthMonitorKey, out HealthMonitor monitor))
return monitor.HealthCheckStatus;
return monitor.LastHealthCheckResponse;
if (_healthChecks.TryGetValue(healthCheck, out HealthCheck existingHealthCheck))
{
@@ -391,10 +409,12 @@ namespace Failover
monitor.Dispose(); //failed to add first
}
return null;
return new HealthCheckResponse(HealthStatus.Unknown);
}
else
{
return new HealthCheckResponse(HealthStatus.Failed, "No such health check: " + healthCheck);
}
return new HealthCheckStatus(false, "No such health check: " + healthCheck, null);
}
#endregion
@@ -410,6 +430,9 @@ namespace Failover
public IReadOnlyDictionary<string, WebHook> WebHooks
{ get { return _webHooks; } }
public IReadOnlyDictionary<NetworkAddress, bool> UnderMaintenance
{ get { return _underMaintenance; } }
public IDnsServer DnsServer
{ get { return _dnsServer; } }

34
Apps/FailoverApp/WebHook.cs
View File

@@ -201,7 +201,7 @@ namespace Failover
ConditionalHttpReload();
}
public Task CallAsync(IPAddress address, string healthCheck, HealthCheckStatus healthCheckStatus)
public Task CallAsync(IPAddress address, string healthCheck, HealthCheckResponse healthCheckResponse)
{
if (!_enabled)
return Task.CompletedTask;
@@ -219,14 +219,17 @@ namespace Failover
jsonWriter.WritePropertyName("healthCheck");
jsonWriter.WriteValue(healthCheck);
jsonWriter.WritePropertyName("isHealthy");
jsonWriter.WriteValue(healthCheckStatus.IsHealthy);
jsonWriter.WritePropertyName("status");
jsonWriter.WriteValue(healthCheckResponse.Status.ToString());
if (healthCheckResponse.Status == HealthStatus.Failed)
{
jsonWriter.WritePropertyName("failureReason");
jsonWriter.WriteValue(healthCheckStatus.FailureReason);
jsonWriter.WriteValue(healthCheckResponse.FailureReason);
}
jsonWriter.WritePropertyName("dateTime");
jsonWriter.WriteValue(healthCheckStatus.DateTime);
jsonWriter.WriteValue(healthCheckResponse.DateTime);
jsonWriter.WriteEndObject();
jsonWriter.Flush();
@@ -257,8 +260,8 @@ namespace Failover
jsonWriter.WritePropertyName("healthCheck");
jsonWriter.WriteValue(healthCheck);
jsonWriter.WritePropertyName("isHealthy");
jsonWriter.WriteValue(false);
jsonWriter.WritePropertyName("status");
jsonWriter.WriteValue("Error");
jsonWriter.WritePropertyName("failureReason");
jsonWriter.WriteValue(ex.ToString());
@@ -277,7 +280,7 @@ namespace Failover
return CallAsync(content);
}
public Task CallAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckStatus healthCheckStatus)
public Task CallAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckResponse healthCheckResponse)
{
if (!_enabled)
return Task.CompletedTask;
@@ -298,14 +301,17 @@ namespace Failover
jsonWriter.WritePropertyName("healthCheck");
jsonWriter.WriteValue(healthCheck);
jsonWriter.WritePropertyName("isHealthy");
jsonWriter.WriteValue(healthCheckStatus.IsHealthy);
jsonWriter.WritePropertyName("status");
jsonWriter.WriteValue(healthCheckResponse.Status.ToString());
if (healthCheckResponse.Status == HealthStatus.Failed)
{
jsonWriter.WritePropertyName("failureReason");
jsonWriter.WriteValue(healthCheckStatus.FailureReason);
jsonWriter.WriteValue(healthCheckResponse.FailureReason);
}
jsonWriter.WritePropertyName("dateTime");
jsonWriter.WriteValue(healthCheckStatus.DateTime);
jsonWriter.WriteValue(healthCheckResponse.DateTime);
jsonWriter.WriteEndObject();
jsonWriter.Flush();
@@ -339,8 +345,8 @@ namespace Failover
jsonWriter.WritePropertyName("healthCheck");
jsonWriter.WriteValue(healthCheck);
jsonWriter.WritePropertyName("isHealthy");
jsonWriter.WriteValue(false);
jsonWriter.WritePropertyName("status");
jsonWriter.WriteValue("Error");
jsonWriter.WritePropertyName("failureReason");
jsonWriter.WriteValue(ex.ToString());

10
Apps/FailoverApp/dnsApp.config
View File

@@ -80,5 +80,15 @@
"https://webhooks.example.com/default"
]
}
],
"underMaintenance": [
{
"network": "192.168.10.2/32",
"enable": false
},
{
"network": "10.1.1.0/24",
"enable": false
}
]
}