diff --git a/Apps/FailoverApp/Address.cs b/Apps/FailoverApp/Address.cs index 96fb3b5e..12172daa 100644 --- a/Apps/FailoverApp/Address.cs +++ b/Apps/FailoverApp/Address.cs @@ -78,11 +78,17 @@ namespace Failover if (address.AddressFamily == AddressFamily.InterNetwork) { - HealthCheckStatus status = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true); - if (status is null) - answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.A, question.Class, 30, new DnsARecord(address))); - else if (status.IsHealthy) - answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.A, question.Class, appRecordTtl, new DnsARecord(address))); + HealthCheckResponse response = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true); + switch (response.Status) + { + case HealthStatus.Unknown: + answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.A, question.Class, 30, new DnsARecord(address))); + break; + + case HealthStatus.Healthy: + answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.A, question.Class, appRecordTtl, new DnsARecord(address))); + break; + } } } break; @@ -94,11 +100,17 @@ namespace Failover if (address.AddressFamily == AddressFamily.InterNetworkV6) { - HealthCheckStatus status = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true); - if (status is null) - answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.AAAA, question.Class, 30, new DnsAAAARecord(address))); - else if (status.IsHealthy) - answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.AAAA, question.Class, appRecordTtl, new DnsAAAARecord(address))); + HealthCheckResponse response = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, true); + switch (response.Status) + { + case HealthStatus.Unknown: + answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.AAAA, question.Class, 30, new DnsAAAARecord(address))); + break; + + case HealthStatus.Healthy: + answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.AAAA, question.Class, appRecordTtl, new DnsAAAARecord(address))); + break; + } } } break; @@ -113,16 +125,12 @@ namespace Failover foreach (dynamic jsonAddress in jsonAddresses) { IPAddress address = IPAddress.Parse(jsonAddress.Value); - HealthCheckStatus status = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, false); + HealthCheckResponse response = _healthService.QueryStatus(address, healthCheck, healthCheckUrl, false); - string text = "app=failover; addressType=" + type.ToString() + "; address=" + address.ToString() + "; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri); + string text = "app=failover; addressType=" + type.ToString() + "; address=" + address.ToString() + "; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri) + "; healthStatus=" + response.Status.ToString() + ";"; - if (status is null) - text += "; healthStatus=Unknown;"; - else if (status.IsHealthy) - text += "; healthStatus=Healthy;"; - else - text += "; healthStatus=Failed; failureReason=" + status.FailureReason + ";"; + if (response.Status == HealthStatus.Failed) + text += " failureReason=" + response.FailureReason + ";"; answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, appRecordTtl, new DnsTXTRecord(text))); } diff --git a/Apps/FailoverApp/CNAME.cs b/Apps/FailoverApp/CNAME.cs index 056798ef..bef47165 100644 --- a/Apps/FailoverApp/CNAME.cs +++ b/Apps/FailoverApp/CNAME.cs @@ -57,20 +57,20 @@ namespace Failover private IReadOnlyList GetAnswers(string domain, DnsQuestionRecord question, string zoneName, uint appRecordTtl, string healthCheck, Uri healthCheckUrl) { - HealthCheckStatus status = _healthService.QueryStatus(domain, question.Type, healthCheck, healthCheckUrl, true); - if (status is null) + HealthCheckResponse response = _healthService.QueryStatus(domain, question.Type, healthCheck, healthCheckUrl, true); + switch (response.Status) { - if (question.Name.Equals(zoneName, StringComparison.OrdinalIgnoreCase)) //check for zone apex - return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.ANAME, DnsClass.IN, 30, new DnsANAMERecord(domain)) }; //use ANAME - else - return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.CNAME, DnsClass.IN, 30, new DnsCNAMERecord(domain)) }; - } - else if (status.IsHealthy) - { - if (question.Name.Equals(zoneName, StringComparison.OrdinalIgnoreCase)) //check for zone apex - return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.ANAME, DnsClass.IN, appRecordTtl, new DnsANAMERecord(domain)) }; //use ANAME - else - return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.CNAME, DnsClass.IN, appRecordTtl, new DnsCNAMERecord(domain)) }; + case HealthStatus.Unknown: + if (question.Name.Equals(zoneName, StringComparison.OrdinalIgnoreCase)) //check for zone apex + return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.ANAME, DnsClass.IN, 30, new DnsANAMERecord(domain)) }; //use ANAME + else + return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.CNAME, DnsClass.IN, 30, new DnsCNAMERecord(domain)) }; + + case HealthStatus.Healthy: + if (question.Name.Equals(zoneName, StringComparison.OrdinalIgnoreCase)) //check for zone apex + return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.ANAME, DnsClass.IN, appRecordTtl, new DnsANAMERecord(domain)) }; //use ANAME + else + return new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.CNAME, DnsClass.IN, appRecordTtl, new DnsCNAMERecord(domain)) }; } return null; @@ -79,31 +79,23 @@ namespace Failover private void GetStatusAnswers(string domain, FailoverType type, DnsQuestionRecord question, uint appRecordTtl, string healthCheck, Uri healthCheckUrl, List answers) { { - HealthCheckStatus status = _healthService.QueryStatus(domain, DnsResourceRecordType.A, healthCheck, healthCheckUrl, false); + HealthCheckResponse response = _healthService.QueryStatus(domain, DnsResourceRecordType.A, healthCheck, healthCheckUrl, false); - string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: A; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri); + string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: A; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri) + "; healthStatus=" + response.Status.ToString() + ";"; - if (status is null) - text += "; healthStatus=Unknown;"; - else if (status.IsHealthy) - text += "; healthStatus=Healthy;"; - else - text += "; healthStatus=Failed; failureReason=" + status.FailureReason + ";"; + if (response.Status == HealthStatus.Failed) + text += " failureReason=" + response.FailureReason + ";"; answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, appRecordTtl, new DnsTXTRecord(text))); } { - HealthCheckStatus status = _healthService.QueryStatus(domain, DnsResourceRecordType.AAAA, healthCheck, healthCheckUrl, false); + HealthCheckResponse response = _healthService.QueryStatus(domain, DnsResourceRecordType.AAAA, healthCheck, healthCheckUrl, false); - string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: AAAA; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri); + string text = "app=failover; cnameType=" + type.ToString() + "; domain=" + domain + "; qType: AAAA; healthCheck=" + healthCheck + (healthCheckUrl is null ? "" : "; healthCheckUrl=" + healthCheckUrl.AbsoluteUri) + "; healthStatus=" + response.Status.ToString() + ";"; - if (status is null) - text += "; healthStatus=Unknown;"; - else if (status.IsHealthy) - text += "; healthStatus=Healthy;"; - else - text += "; healthStatus=Failed; failureReason=" + status.FailureReason + ";"; + if (response.Status == HealthStatus.Failed) + text += " failureReason=" + response.FailureReason + ";"; answers.Add(new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, appRecordTtl, new DnsTXTRecord(text))); } diff --git a/Apps/FailoverApp/EmailAlert.cs b/Apps/FailoverApp/EmailAlert.cs index db0c6b91..a7a28fca 100644 --- a/Apps/FailoverApp/EmailAlert.cs +++ b/Apps/FailoverApp/EmailAlert.cs @@ -188,7 +188,7 @@ namespace Failover _smtpClient.Proxy = _service.DnsServer.Proxy; } - public Task SendAlertAsync(IPAddress address, string healthCheck, HealthCheckStatus healthCheckStatus) + public Task SendAlertAsync(IPAddress address, string healthCheck, HealthCheckResponse healthCheckResponse) { if (!_enabled || (_mailFrom is null) || (_alertTo is null) || (_alertTo.Length == 0)) return Task.CompletedTask; @@ -200,32 +200,40 @@ namespace Failover foreach (MailAddress alertTo in _alertTo) message.To.Add(alertTo); - if (healthCheckStatus.IsHealthy) + message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is " + healthCheckResponse.Status.ToString().ToUpper(); + + switch (healthCheckResponse.Status) { - message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is Healthy"; - message.Body = @"Hi, - -The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"] and found that the address was healthy. - -Alert time: " + healthCheckStatus.DateTime.ToString("R") + @" - -Regards, -DNS Failover App -"; - } - else - { - message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is Failed"; - message.Body = @"Hi, + case HealthStatus.Failed: + message.Body = @"Hi, The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"] and found that the address failed to respond. -The failure reason is: " + healthCheckStatus.FailureReason + @" -Alert time: " + healthCheckStatus.DateTime.ToString("R") + @" +Address: " + address.ToString() + @" +Health Check: " + healthCheck + @" +Status: " + healthCheckResponse.Status.ToString().ToUpper() + @" +Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @" +Failure Reason: " + healthCheckResponse.FailureReason + @" Regards, DNS Failover App "; + break; + + default: + message.Body = @"Hi, + +The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"] and found that the address status was " + healthCheckResponse.Status.ToString().ToUpper() + @". + +Address: " + address.ToString() + @" +Health Check: " + healthCheck + @" +Status: " + healthCheckResponse.Status.ToString().ToUpper() + @" +Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @" + +Regards, +DNS Failover App +"; + break; } return SendMailAsync(message); @@ -243,13 +251,16 @@ DNS Failover App foreach (MailAddress alertTo in _alertTo) message.To.Add(alertTo); - message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is Error"; + message.Subject = "[Alert] Address [" + address.ToString() + "] Status Is ERROR"; message.Body = @"Hi, The DNS Failover App has failed to perform a health check [" + healthCheck + "] on the address [" + address.ToString() + @"]. -The error description is: " + ex.ToString() + @" -Alert time: " + DateTime.UtcNow.ToString("R") + @" +Address: " + address.ToString() + @" +Health Check: " + healthCheck + @" +Status: ERROR +Alert Time: " + DateTime.UtcNow.ToString("R") + @" +Failure Reason: " + ex.ToString() + @" Regards, DNS Failover App @@ -258,7 +269,7 @@ DNS Failover App return SendMailAsync(message); } - public Task SendAlertAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckStatus healthCheckStatus) + public Task SendAlertAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckResponse healthCheckResponse) { if (!_enabled || (_mailFrom is null) || (_alertTo is null) || (_alertTo.Length == 0)) return Task.CompletedTask; @@ -270,34 +281,42 @@ DNS Failover App foreach (MailAddress alertTo in _alertTo) message.To.Add(alertTo); - if (healthCheckStatus.IsHealthy) + message.Subject = "[Alert] Domain [" + domain + "] Status Is " + healthCheckResponse.Status.ToString().ToUpper(); + + switch(healthCheckResponse.Status) { - message.Subject = "[Alert] Domain [" + domain + "] Status Is Healthy"; - message.Body = @"Hi, - -The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"] and found that the domain name was healthy. - -DNS record type: " + type.ToString() + @" -Alert time: " + healthCheckStatus.DateTime.ToString("R") + @" - -Regards, -DNS Failover App -"; - } - else - { - message.Subject = "[Alert] Domain [" + domain + "] Status Is Failed"; - message.Body = @"Hi, + case HealthStatus.Failed: + message.Body = @"Hi, The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"] and found that the domain name failed to respond. -The failure reason is: " + healthCheckStatus.FailureReason + @" -DNS record type: " + type.ToString() + @" -Alert time: " + healthCheckStatus.DateTime.ToString("R") + @" +Domain: " + domain + @" +Record Type: " + type.ToString() + @" +Health Check: " + healthCheck + @" +Status: " + healthCheckResponse.Status.ToString().ToUpper() + @" +Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @" +Failure Reason: " + healthCheckResponse.FailureReason + @" Regards, DNS Failover App "; + break; + + default: + message.Body = @"Hi, + +The DNS Failover App was successfully able to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"] and found that the domain name status was " + healthCheckResponse.Status.ToString().ToUpper() + @". + +Domain: " + domain + @" +Record Type: " + type.ToString() + @" +Health Check: " + healthCheck + @" +Status: " + healthCheckResponse.Status.ToString().ToUpper() + @" +Alert Time: " + healthCheckResponse.DateTime.ToString("R") + @" + +Regards, +DNS Failover App +"; + break; } return SendMailAsync(message); @@ -315,14 +334,17 @@ DNS Failover App foreach (MailAddress alertTo in _alertTo) message.To.Add(alertTo); - message.Subject = "[Alert] Domain [" + domain + "] Status Is Error"; + message.Subject = "[Alert] Domain [" + domain + "] Status Is ERROR"; message.Body = @"Hi, The DNS Failover App has failed to perform a health check [" + healthCheck + "] on the domain name [" + domain + @"]. -The error description is: " + ex.ToString() + @" -DNS record type: " + type.ToString() + @" -Alert time: " + DateTime.UtcNow.ToString("R") + @" +Domain: " + domain + @" +Record Type: " + type.ToString() + @" +Health Check: " + healthCheck + @" +Status: ERROR +Alert Time: " + DateTime.UtcNow.ToString("R") + @" +Failure Reason: " + ex.ToString() + @" Regards, DNS Failover App diff --git a/Apps/FailoverApp/FailoverApp.csproj b/Apps/FailoverApp/FailoverApp.csproj index f75b9d1a..a7f47075 100644 --- a/Apps/FailoverApp/FailoverApp.csproj +++ b/Apps/FailoverApp/FailoverApp.csproj @@ -4,7 +4,7 @@ net5.0 false true - 1.4 + 1.6 Technitium Technitium DNS Server Shreyas Zare diff --git a/Apps/FailoverApp/HealthCheck.cs b/Apps/FailoverApp/HealthCheck.cs index 151a44fa..82d53e43 100644 --- a/Apps/FailoverApp/HealthCheck.cs +++ b/Apps/FailoverApp/HealthCheck.cs @@ -25,6 +25,7 @@ using System.Net.NetworkInformation; using System.Net.Sockets; using System.Threading.Tasks; using TechnitiumLibrary.IO; +using TechnitiumLibrary.Net; using TechnitiumLibrary.Net.Dns; using TechnitiumLibrary.Net.Proxy; @@ -258,7 +259,7 @@ namespace Failover ConditionalHttpReload(); } - public async Task IsHealthyAsync(string domain, DnsResourceRecordType type, Uri healthCheckUrl) + public async Task IsHealthyAsync(string domain, DnsResourceRecordType type, Uri healthCheckUrl) { switch (type) { @@ -266,57 +267,68 @@ namespace Failover { DnsDatagram response = await _service.DnsServer.DirectQueryAsync(new DnsQuestionRecord(domain, type, DnsClass.IN)); if ((response is null) || (response.Answer.Count == 0)) - return new HealthCheckStatus(false, "Failed to resolve address.", null); + return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address."); IReadOnlyList addresses = DnsClient.ParseResponseA(response); if (addresses.Count > 0) { - HealthCheckStatus lastStatus = null; + HealthCheckResponse lastResponse = null; foreach (IPAddress address in addresses) { - lastStatus = await IsHealthyAsync(address, healthCheckUrl); - if (lastStatus.IsHealthy) - return lastStatus; + lastResponse = await IsHealthyAsync(address, healthCheckUrl); + if (lastResponse.Status == HealthStatus.Healthy) + return lastResponse; } - return lastStatus; + return lastResponse; } - return new HealthCheckStatus(false, "Failed to resolve address.", null); + return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address."); } case DnsResourceRecordType.AAAA: { DnsDatagram response = await _service.DnsServer.DirectQueryAsync(new DnsQuestionRecord(domain, type, DnsClass.IN)); if ((response is null) || (response.Answer.Count == 0)) - return new HealthCheckStatus(false, "Failed to resolve address.", null); + return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address."); IReadOnlyList addresses = DnsClient.ParseResponseAAAA(response); if (addresses.Count > 0) { - HealthCheckStatus lastStatus = null; + HealthCheckResponse lastResponse = null; foreach (IPAddress address in addresses) { - lastStatus = await IsHealthyAsync(address, healthCheckUrl); - if (lastStatus.IsHealthy) - return lastStatus; + lastResponse = await IsHealthyAsync(address, healthCheckUrl); + if (lastResponse.Status == HealthStatus.Healthy) + return lastResponse; } - return lastStatus; + return lastResponse; } - return new HealthCheckStatus(false, "Failed to resolve address.", null); + return new HealthCheckResponse(HealthStatus.Failed, "Failed to resolve address."); } default: - return new HealthCheckStatus(false, "Not supported.", null); + return new HealthCheckResponse(HealthStatus.Failed, "Not supported."); } } - public async Task IsHealthyAsync(IPAddress address, Uri healthCheckUrl) + public async Task IsHealthyAsync(IPAddress address, Uri healthCheckUrl) { + foreach (KeyValuePair network in _service.UnderMaintenance) + { + if (network.Key.Contains(address)) + { + if (network.Value) + return new HealthCheckResponse(HealthStatus.Maintenance); + + break; + } + } + switch (_type) { case HealthCheckType.Ping: @@ -332,13 +344,13 @@ namespace Failover { PingReply reply = await ping.SendPingAsync(address, _timeout); if (reply.Status == IPStatus.Success) - return new HealthCheckStatus(true, null, null); + return new HealthCheckResponse(HealthStatus.Healthy); lastReason = reply.Status.ToString(); } while (++retry < _retries); - return new HealthCheckStatus(false, lastReason, null); + return new HealthCheckResponse(HealthStatus.Failed, lastReason); } } @@ -368,7 +380,7 @@ namespace Failover } } - return new HealthCheckStatus(true, null, null); + return new HealthCheckResponse(HealthStatus.Healthy); } catch (TimeoutException ex) { @@ -387,7 +399,7 @@ namespace Failover } while (++retry < _retries); - return new HealthCheckStatus(false, lastReason, lastException); + return new HealthCheckResponse(HealthStatus.Failed, lastReason, lastException); } case HealthCheckType.Http: @@ -410,7 +422,7 @@ namespace Failover url = _url; if (url is null) - return new HealthCheckStatus(false, "Missing health check URL in APP record as well as in app config.", null); + return new HealthCheckResponse(HealthStatus.Failed, "Missing health check URL in APP record as well as in app config."); if (_type == HealthCheckType.Http) { @@ -434,9 +446,9 @@ namespace Failover HttpResponseMessage httpResponse = await _httpClient.SendAsync(httpRequest); if (httpResponse.IsSuccessStatusCode) - return new HealthCheckStatus(true, null, null); + return new HealthCheckResponse(HealthStatus.Healthy); - return new HealthCheckStatus(false, "Received HTTP status code: " + (int)httpResponse.StatusCode + " " + httpResponse.StatusCode.ToString() + "; URL: " + url.AbsoluteUri, null); + return new HealthCheckResponse(HealthStatus.Failed, "Received HTTP status code: " + (int)httpResponse.StatusCode + " " + httpResponse.StatusCode.ToString() + "; URL: " + url.AbsoluteUri); } catch (TaskCanceledException ex) { @@ -455,7 +467,7 @@ namespace Failover } while (++retry < _retries); - return new HealthCheckStatus(false, lastReason, lastException); + return new HealthCheckResponse(HealthStatus.Failed, lastReason, lastException); } default: diff --git a/Apps/FailoverApp/HealthCheckStatus.cs b/Apps/FailoverApp/HealthCheckResponse.cs similarity index 76% rename from Apps/FailoverApp/HealthCheckStatus.cs rename to Apps/FailoverApp/HealthCheckResponse.cs index c0afcfff..84e7f806 100644 --- a/Apps/FailoverApp/HealthCheckStatus.cs +++ b/Apps/FailoverApp/HealthCheckResponse.cs @@ -21,12 +21,20 @@ using System; namespace Failover { - class HealthCheckStatus + enum HealthStatus + { + Unknown = 0, + Failed = 1, + Healthy = 2, + Maintenance = 3 + } + + class HealthCheckResponse { #region variables public readonly DateTime DateTime = DateTime.UtcNow; - public readonly bool IsHealthy; + public readonly HealthStatus Status; public readonly string FailureReason; public readonly Exception Exception; @@ -34,9 +42,9 @@ namespace Failover #region constructor - public HealthCheckStatus(bool isHealthy, string failureReason, Exception exception) + public HealthCheckResponse(HealthStatus status, string failureReason = null, Exception exception = null) { - IsHealthy = isHealthy; + Status = status; FailureReason = failureReason; Exception = exception; } diff --git a/Apps/FailoverApp/HealthMonitor.cs b/Apps/FailoverApp/HealthMonitor.cs index ead71224..36544660 100644 --- a/Apps/FailoverApp/HealthMonitor.cs +++ b/Apps/FailoverApp/HealthMonitor.cs @@ -38,10 +38,10 @@ namespace Failover readonly Timer _healthCheckTimer; const int HEALTH_CHECK_TIMER_INITIAL_INTERVAL = 1000; - HealthCheckStatus _healthCheckStatus; + HealthCheckResponse _lastHealthCheckResponse; const int MONITOR_EXPIRY = 1 * 60 * 60 * 1000; //1 hour - DateTime _lastStatusCheckedOn; + DateTime _lastHealthStatusCheckedOn; #endregion @@ -59,52 +59,77 @@ namespace Failover { if (_healthCheck is null) { - _healthCheckStatus = null; + _lastHealthCheckResponse = null; } else { - HealthCheckStatus healthCheckStatus = await _healthCheck.IsHealthyAsync(_address, healthCheckUrl); + HealthCheckResponse healthCheckResponse = await _healthCheck.IsHealthyAsync(_address, healthCheckUrl); - bool sendAlert = false; + bool statusChanged = false; + bool maintenance = false; - if (_healthCheckStatus is null) + if (_lastHealthCheckResponse is null) { - if (!healthCheckStatus.IsHealthy) - sendAlert = true; + switch (healthCheckResponse.Status) + { + case HealthStatus.Failed: + statusChanged = true; + break; + + case HealthStatus.Maintenance: + statusChanged = true; + maintenance = true; + break; + } } else { - if (_healthCheckStatus.IsHealthy != healthCheckStatus.IsHealthy) - sendAlert = true; + if (_lastHealthCheckResponse.Status != healthCheckResponse.Status) + { + statusChanged = true; + + if ((_lastHealthCheckResponse.Status == HealthStatus.Maintenance) || (healthCheckResponse.Status == HealthStatus.Maintenance)) + maintenance = true; + } } - if (sendAlert) + if (statusChanged) { - if (healthCheckStatus.IsHealthy) - _dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is HEALTHY based on '" + _healthCheck.Name + "' health check."); - else - _dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckStatus.FailureReason); + switch (healthCheckResponse.Status) + { + case HealthStatus.Failed: + _dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckResponse.FailureReason); + break; - if (healthCheckStatus.Exception is not null) - _dnsServer.WriteLog(healthCheckStatus.Exception); + default: + _dnsServer.WriteLog("ALERT! Address [" + _address.ToString() + "] status is " + healthCheckResponse.Status.ToString().ToUpper() + " based on '" + _healthCheck.Name + "' health check."); + break; + } - EmailAlert emailAlert = _healthCheck.EmailAlert; - if (emailAlert is not null) - _ = emailAlert.SendAlertAsync(_address, _healthCheck.Name, healthCheckStatus); + if (healthCheckResponse.Exception is not null) + _dnsServer.WriteLog(healthCheckResponse.Exception); + + if (!maintenance) + { + //avoid sending email alerts when switching from or to maintenance + EmailAlert emailAlert = _healthCheck.EmailAlert; + if (emailAlert is not null) + _ = emailAlert.SendAlertAsync(_address, _healthCheck.Name, healthCheckResponse); + } WebHook webHook = _healthCheck.WebHook; if (webHook is not null) - _ = webHook.CallAsync(_address, _healthCheck.Name, healthCheckStatus); + _ = webHook.CallAsync(_address, _healthCheck.Name, healthCheckResponse); } - _healthCheckStatus = healthCheckStatus; + _lastHealthCheckResponse = healthCheckResponse; } } catch (Exception ex) { _dnsServer.WriteLog(ex); - if (_healthCheckStatus is null) + if (_lastHealthCheckResponse is null) { EmailAlert emailAlert = _healthCheck.EmailAlert; if (emailAlert is not null) @@ -114,11 +139,11 @@ namespace Failover if (webHook is not null) _ = webHook.CallAsync(_address, _healthCheck.Name, ex); - _healthCheckStatus = new HealthCheckStatus(false, ex.ToString(), ex); + _lastHealthCheckResponse = new HealthCheckResponse(HealthStatus.Failed, ex.ToString(), ex); } else { - _healthCheckStatus = null; + _lastHealthCheckResponse = null; } } finally @@ -144,52 +169,77 @@ namespace Failover { if (_healthCheck is null) { - _healthCheckStatus = null; + _lastHealthCheckResponse = null; } else { - HealthCheckStatus healthCheckStatus = await _healthCheck.IsHealthyAsync(_domain, _type, healthCheckUrl); + HealthCheckResponse healthCheckResponse = await _healthCheck.IsHealthyAsync(_domain, _type, healthCheckUrl); - bool sendAlert = false; + bool statusChanged = false; + bool maintenance = false; - if (_healthCheckStatus is null) + if (_lastHealthCheckResponse is null) { - if (!healthCheckStatus.IsHealthy) - sendAlert = true; + switch (healthCheckResponse.Status) + { + case HealthStatus.Failed: + statusChanged = true; + break; + + case HealthStatus.Maintenance: + statusChanged = true; + maintenance = true; + break; + } } else { - if (_healthCheckStatus.IsHealthy != healthCheckStatus.IsHealthy) - sendAlert = true; + if (_lastHealthCheckResponse.Status != healthCheckResponse.Status) + { + statusChanged = true; + + if ((_lastHealthCheckResponse.Status == HealthStatus.Maintenance) || (healthCheckResponse.Status == HealthStatus.Maintenance)) + maintenance = true; + } } - if (sendAlert) + if (statusChanged) { - if (healthCheckStatus.IsHealthy) - _dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is HEALTHY based on '" + _healthCheck.Name + "' health check."); - else - _dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckStatus.FailureReason); + switch (healthCheckResponse.Status) + { + case HealthStatus.Failed: + _dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is FAILED based on '" + _healthCheck.Name + "' health check. The failure reason is: " + healthCheckResponse.FailureReason); + break; - if (healthCheckStatus.Exception is not null) - _dnsServer.WriteLog(healthCheckStatus.Exception); + default: + _dnsServer.WriteLog("ALERT! Domain [" + _domain + "] type [" + _type.ToString() + "] status is " + healthCheckResponse.Status.ToString().ToUpper() + " based on '" + _healthCheck.Name + "' health check."); + break; + } - EmailAlert emailAlert = _healthCheck.EmailAlert; - if (emailAlert is not null) - _ = emailAlert.SendAlertAsync(_domain, _type, _healthCheck.Name, healthCheckStatus); + if (healthCheckResponse.Exception is not null) + _dnsServer.WriteLog(healthCheckResponse.Exception); + + if (!maintenance) + { + //avoid sending email alerts when switching from or to maintenance + EmailAlert emailAlert = _healthCheck.EmailAlert; + if (emailAlert is not null) + _ = emailAlert.SendAlertAsync(_domain, _type, _healthCheck.Name, healthCheckResponse); + } WebHook webHook = _healthCheck.WebHook; if (webHook is not null) - _ = webHook.CallAsync(_domain, _type, _healthCheck.Name, healthCheckStatus); + _ = webHook.CallAsync(_domain, _type, _healthCheck.Name, healthCheckResponse); } - _healthCheckStatus = healthCheckStatus; + _lastHealthCheckResponse = healthCheckResponse; } } catch (Exception ex) { _dnsServer.WriteLog(ex); - if (_healthCheckStatus is null) + if (_lastHealthCheckResponse is null) { EmailAlert emailAlert = _healthCheck.EmailAlert; if (emailAlert is not null) @@ -199,11 +249,11 @@ namespace Failover if (webHook is not null) _ = webHook.CallAsync(_domain, _type, _healthCheck.Name, ex); - _healthCheckStatus = new HealthCheckStatus(false, ex.ToString(), ex); + _lastHealthCheckResponse = new HealthCheckResponse(HealthStatus.Failed, ex.ToString(), ex); } else { - _healthCheckStatus = null; + _lastHealthCheckResponse = null; } } finally @@ -248,19 +298,19 @@ namespace Failover public bool IsExpired() { - return DateTime.UtcNow > _lastStatusCheckedOn.AddMilliseconds(MONITOR_EXPIRY); + return DateTime.UtcNow > _lastHealthStatusCheckedOn.AddMilliseconds(MONITOR_EXPIRY); } #endregion #region properties - public HealthCheckStatus HealthCheckStatus + public HealthCheckResponse LastHealthCheckResponse { get { - _lastStatusCheckedOn = DateTime.UtcNow; - return _healthCheckStatus; + _lastHealthStatusCheckedOn = DateTime.UtcNow; + return _lastHealthCheckResponse; } } diff --git a/Apps/FailoverApp/HealthService.cs b/Apps/FailoverApp/HealthService.cs index 36863425..a5d4e41a 100644 --- a/Apps/FailoverApp/HealthService.cs +++ b/Apps/FailoverApp/HealthService.cs @@ -23,6 +23,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; using System.Threading; +using TechnitiumLibrary.Net; using TechnitiumLibrary.Net.Dns; namespace Failover @@ -38,6 +39,7 @@ namespace Failover readonly ConcurrentDictionary _healthChecks = new ConcurrentDictionary(1, 5); readonly ConcurrentDictionary _emailAlerts = new ConcurrentDictionary(1, 2); readonly ConcurrentDictionary _webHooks = new ConcurrentDictionary(1, 2); + readonly ConcurrentDictionary _underMaintenance = new ConcurrentDictionary(); readonly ConcurrentDictionary _healthMonitors = new ConcurrentDictionary(); @@ -347,14 +349,28 @@ namespace Failover } } } + + //under maintenance networks + _underMaintenance.Clear(); + + if (jsonConfig.underMaintenance is not null) + { + foreach (dynamic jsonNetwork in jsonConfig.underMaintenance) + { + string network = jsonNetwork.network.Value; + bool enable = jsonNetwork.enable.Value; + + _underMaintenance.TryAdd(NetworkAddress.Parse(network), enable); + } + } } - public HealthCheckStatus QueryStatus(IPAddress address, string healthCheck, Uri healthCheckUrl, bool tryAdd) + public HealthCheckResponse QueryStatus(IPAddress address, string healthCheck, Uri healthCheckUrl, bool tryAdd) { string healthMonitorKey = GetHealthMonitorKey(address, healthCheck, healthCheckUrl); if (_healthMonitors.TryGetValue(healthMonitorKey, out HealthMonitor monitor)) - return monitor.HealthCheckStatus; + return monitor.LastHealthCheckResponse; if (_healthChecks.TryGetValue(healthCheck, out HealthCheck existingHealthCheck)) { @@ -366,20 +382,22 @@ namespace Failover monitor.Dispose(); //failed to add first } - return null; + return new HealthCheckResponse(HealthStatus.Unknown); + } + else + { + return new HealthCheckResponse(HealthStatus.Failed, "No such health check: " + healthCheck); } - - return new HealthCheckStatus(false, "No such health check: " + healthCheck, null); } - public HealthCheckStatus QueryStatus(string domain, DnsResourceRecordType type, string healthCheck, Uri healthCheckUrl, bool tryAdd) + public HealthCheckResponse QueryStatus(string domain, DnsResourceRecordType type, string healthCheck, Uri healthCheckUrl, bool tryAdd) { domain = domain.ToLower(); string healthMonitorKey = GetHealthMonitorKey(domain, type, healthCheck, healthCheckUrl); if (_healthMonitors.TryGetValue(healthMonitorKey, out HealthMonitor monitor)) - return monitor.HealthCheckStatus; + return monitor.LastHealthCheckResponse; if (_healthChecks.TryGetValue(healthCheck, out HealthCheck existingHealthCheck)) { @@ -391,10 +409,12 @@ namespace Failover monitor.Dispose(); //failed to add first } - return null; + return new HealthCheckResponse(HealthStatus.Unknown); + } + else + { + return new HealthCheckResponse(HealthStatus.Failed, "No such health check: " + healthCheck); } - - return new HealthCheckStatus(false, "No such health check: " + healthCheck, null); } #endregion @@ -410,6 +430,9 @@ namespace Failover public IReadOnlyDictionary WebHooks { get { return _webHooks; } } + public IReadOnlyDictionary UnderMaintenance + { get { return _underMaintenance; } } + public IDnsServer DnsServer { get { return _dnsServer; } } diff --git a/Apps/FailoverApp/WebHook.cs b/Apps/FailoverApp/WebHook.cs index 8ee5b1b1..f69dbcfc 100644 --- a/Apps/FailoverApp/WebHook.cs +++ b/Apps/FailoverApp/WebHook.cs @@ -201,7 +201,7 @@ namespace Failover ConditionalHttpReload(); } - public Task CallAsync(IPAddress address, string healthCheck, HealthCheckStatus healthCheckStatus) + public Task CallAsync(IPAddress address, string healthCheck, HealthCheckResponse healthCheckResponse) { if (!_enabled) return Task.CompletedTask; @@ -219,14 +219,17 @@ namespace Failover jsonWriter.WritePropertyName("healthCheck"); jsonWriter.WriteValue(healthCheck); - jsonWriter.WritePropertyName("isHealthy"); - jsonWriter.WriteValue(healthCheckStatus.IsHealthy); + jsonWriter.WritePropertyName("status"); + jsonWriter.WriteValue(healthCheckResponse.Status.ToString()); - jsonWriter.WritePropertyName("failureReason"); - jsonWriter.WriteValue(healthCheckStatus.FailureReason); + if (healthCheckResponse.Status == HealthStatus.Failed) + { + jsonWriter.WritePropertyName("failureReason"); + jsonWriter.WriteValue(healthCheckResponse.FailureReason); + } jsonWriter.WritePropertyName("dateTime"); - jsonWriter.WriteValue(healthCheckStatus.DateTime); + jsonWriter.WriteValue(healthCheckResponse.DateTime); jsonWriter.WriteEndObject(); jsonWriter.Flush(); @@ -257,8 +260,8 @@ namespace Failover jsonWriter.WritePropertyName("healthCheck"); jsonWriter.WriteValue(healthCheck); - jsonWriter.WritePropertyName("isHealthy"); - jsonWriter.WriteValue(false); + jsonWriter.WritePropertyName("status"); + jsonWriter.WriteValue("Error"); jsonWriter.WritePropertyName("failureReason"); jsonWriter.WriteValue(ex.ToString()); @@ -277,7 +280,7 @@ namespace Failover return CallAsync(content); } - public Task CallAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckStatus healthCheckStatus) + public Task CallAsync(string domain, DnsResourceRecordType type, string healthCheck, HealthCheckResponse healthCheckResponse) { if (!_enabled) return Task.CompletedTask; @@ -298,14 +301,17 @@ namespace Failover jsonWriter.WritePropertyName("healthCheck"); jsonWriter.WriteValue(healthCheck); - jsonWriter.WritePropertyName("isHealthy"); - jsonWriter.WriteValue(healthCheckStatus.IsHealthy); + jsonWriter.WritePropertyName("status"); + jsonWriter.WriteValue(healthCheckResponse.Status.ToString()); - jsonWriter.WritePropertyName("failureReason"); - jsonWriter.WriteValue(healthCheckStatus.FailureReason); + if (healthCheckResponse.Status == HealthStatus.Failed) + { + jsonWriter.WritePropertyName("failureReason"); + jsonWriter.WriteValue(healthCheckResponse.FailureReason); + } jsonWriter.WritePropertyName("dateTime"); - jsonWriter.WriteValue(healthCheckStatus.DateTime); + jsonWriter.WriteValue(healthCheckResponse.DateTime); jsonWriter.WriteEndObject(); jsonWriter.Flush(); @@ -339,8 +345,8 @@ namespace Failover jsonWriter.WritePropertyName("healthCheck"); jsonWriter.WriteValue(healthCheck); - jsonWriter.WritePropertyName("isHealthy"); - jsonWriter.WriteValue(false); + jsonWriter.WritePropertyName("status"); + jsonWriter.WriteValue("Error"); jsonWriter.WritePropertyName("failureReason"); jsonWriter.WriteValue(ex.ToString()); diff --git a/Apps/FailoverApp/dnsApp.config b/Apps/FailoverApp/dnsApp.config index bb81faf2..d93ffe9f 100644 --- a/Apps/FailoverApp/dnsApp.config +++ b/Apps/FailoverApp/dnsApp.config @@ -80,5 +80,15 @@ "https://webhooks.example.com/default" ] } + ], + "underMaintenance": [ + { + "network": "192.168.10.2/32", + "enable": false + }, + { + "network": "10.1.1.0/24", + "enable": false + } ] } \ No newline at end of file