From ff8203ee70bf1406f4eed34526043896afb932ad Mon Sep 17 00:00:00 2001 From: Shreyas Zare Date: Sat, 12 Nov 2022 13:34:37 +0530 Subject: [PATCH] AdvancedBlocking: refactored complete app design to use less memory when same block lists are used in multiple groups. --- Apps/AdvancedBlockingApp/App.cs | 1990 ++++++++++++++----------------- 1 file changed, 909 insertions(+), 1081 deletions(-) diff --git a/Apps/AdvancedBlockingApp/App.cs b/Apps/AdvancedBlockingApp/App.cs index 00dd625d..1cd2e73d 100644 --- a/Apps/AdvancedBlockingApp/App.cs +++ b/Apps/AdvancedBlockingApp/App.cs @@ -22,7 +22,6 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Net; using System.Net.Http; using System.Net.Sockets; @@ -42,7 +41,6 @@ namespace AdvancedBlocking #region variables IDnsServer _dnsServer; - string _localCacheFolder; DnsSOARecordData _soaRecord; DnsNSRecordData _nsRecord; @@ -53,10 +51,17 @@ namespace AdvancedBlocking IReadOnlyDictionary _networkGroupMap; IReadOnlyDictionary _groups; + IReadOnlyDictionary _allAllowListZones = new Dictionary(0); + IReadOnlyDictionary _allBlockListZones = new Dictionary(0); + + IReadOnlyDictionary _allRegexAllowListZones = new Dictionary(0); + IReadOnlyDictionary _allRegexBlockListZones = new Dictionary(0); + + IReadOnlyDictionary _allAdBlockListZones = new Dictionary(0); + Timer _blockListUrlUpdateTimer; DateTime _blockListUrlLastUpdatedOn; - const int BLOCK_LIST_UPDATE_TIMER_INITIAL_INTERVAL = 5000; - const int BLOCK_LIST_UPDATE_TIMER_PERIODIC_INTERVAL = 900000; + const int BLOCK_LIST_UPDATE_TIMER_INTERVAL = 900000; #endregion @@ -95,692 +100,31 @@ namespace AdvancedBlocking } } - private void FindAndSetBlockListUrlLastUpdatedOn() - { - try - { - string[] files = Directory.GetFiles(_localCacheFolder); - DateTime latest = DateTime.MinValue; - - foreach (string file in files) - { - DateTime lastModified = File.GetLastWriteTimeUtc(file); - - if (lastModified > latest) - latest = lastModified; - } - - _blockListUrlLastUpdatedOn = latest; - } - catch (Exception ex) - { - _dnsServer.WriteLog(ex); - } - } - - private string GetListFilePath(Uri listUrl) - { - using (HashAlgorithm hash = SHA256.Create()) - { - return Path.Combine(_localCacheFolder, Convert.ToHexString(hash.ComputeHash(Encoding.UTF8.GetBytes(listUrl.AbsoluteUri))).ToLower()); - } - } - private async Task UpdateAllListsAsync() { - List downloadedAllowListUrls = new List(); - List downloadedBlockListUrls = new List(); - List downloadedRegexAllowListUrls = new List(); - List downloadedRegexBlockListUrls = new List(); - List downloadedAdblockListUrls = new List(); - bool notModified = false; + List> updateTasks = new List>(); - async Task DownloadListUrlAsync(Uri listUrl, bool isAllowList, bool isRegexList, bool isAdblockList) + foreach (KeyValuePair allAllowListZone in _allAllowListZones) + updateTasks.Add(allAllowListZone.Value.UpdateAsync()); + + foreach (KeyValuePair allBlockListZone in _allBlockListZones) + updateTasks.Add(allBlockListZone.Value.UpdateAsync()); + + foreach (KeyValuePair allRegexAllowListZone in _allRegexAllowListZones) + updateTasks.Add(allRegexAllowListZone.Value.UpdateAsync()); + + foreach (KeyValuePair allRegexBlockListZone in _allRegexBlockListZones) + updateTasks.Add(allRegexBlockListZone.Value.UpdateAsync()); + + foreach (KeyValuePair allAdBlockListZone in _allAdBlockListZones) + updateTasks.Add(allAdBlockListZone.Value.UpdateAsync()); + + await Task.WhenAll(updateTasks); + + foreach (Task updateTask in updateTasks) { - string listFilePath = GetListFilePath(listUrl); - string listDownloadFilePath = listFilePath + ".downloading"; - - try - { - if (File.Exists(listDownloadFilePath)) - File.Delete(listDownloadFilePath); - - SocketsHttpHandler handler = new SocketsHttpHandler(); - handler.Proxy = _dnsServer.Proxy; - handler.UseProxy = _dnsServer.Proxy is not null; - handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; - - using (HttpClient http = new HttpClient(handler)) - { - if (File.Exists(listFilePath)) - http.DefaultRequestHeaders.IfModifiedSince = File.GetLastWriteTimeUtc(listFilePath); - - HttpResponseMessage httpResponse = await http.GetAsync(listUrl); - switch (httpResponse.StatusCode) - { - case HttpStatusCode.OK: - { - using (FileStream fS = new FileStream(listDownloadFilePath, FileMode.Create, FileAccess.Write)) - { - using (Stream httpStream = await httpResponse.Content.ReadAsStreamAsync()) - { - await httpStream.CopyToAsync(fS); - } - } - - if (File.Exists(listFilePath)) - File.Delete(listFilePath); - - File.Move(listDownloadFilePath, listFilePath); - - if (httpResponse.Content.Headers.LastModified != null) - File.SetLastWriteTimeUtc(listFilePath, httpResponse.Content.Headers.LastModified.Value.UtcDateTime); - - if (isAdblockList) - { - lock (downloadedAdblockListUrls) - { - downloadedAdblockListUrls.Add(listUrl); - } - } - else - { - if (isAllowList) - { - if (isRegexList) - { - lock (downloadedRegexAllowListUrls) - { - downloadedRegexAllowListUrls.Add(listUrl); - } - } - else - { - lock (downloadedAllowListUrls) - { - downloadedAllowListUrls.Add(listUrl); - } - } - } - else - { - if (isRegexList) - { - lock (downloadedRegexBlockListUrls) - { - downloadedRegexBlockListUrls.Add(listUrl); - } - } - else - { - lock (downloadedBlockListUrls) - { - downloadedBlockListUrls.Add(listUrl); - } - } - } - } - - _dnsServer.WriteLog("Advanced Blocking app successfully downloaded " + (isAdblockList ? "adblock" : (isRegexList ? "regex " : "") + (isAllowList ? "allow" : "block")) + " list (" + WebUtilities.GetFormattedSize(new FileInfo(listFilePath).Length) + "): " + listUrl.AbsoluteUri); - } - break; - - case HttpStatusCode.NotModified: - { - notModified = true; - - _dnsServer.WriteLog("Advanced Blocking app successfully checked for a new update of the " + (isAdblockList ? "adblock" : (isRegexList ? "regex " : "") + (isAllowList ? "allow" : "block")) + " list: " + listUrl.AbsoluteUri); - } - break; - - default: - throw new HttpRequestException((int)httpResponse.StatusCode + " " + httpResponse.ReasonPhrase); - } - } - } - catch (Exception ex) - { - _dnsServer.WriteLog("Advanced Blocking app failed to download " + (isAdblockList ? "adblock" : (isRegexList ? "regex " : "") + (isAllowList ? "allow" : "block")) + " list and will use previously downloaded file (if available): " + listUrl.AbsoluteUri + "\r\n" + ex.ToString()); - } - } - - List tasks = new List(); - IReadOnlyList uniqueAllowListUrls = GetUniqueAllowListUrls(); - IReadOnlyList uniqueBlockListUrls = GetUniqueBlockListUrls(); - IReadOnlyList uniqueRegexAllowListUrls = GetUniqueRegexAllowListUrls(); - IReadOnlyList uniqueRegexBlockListUrls = GetUniqueRegexBlockListUrls(); - IReadOnlyList uniqueAdblockListUrls = GetUniqueAdblockListUrls(); - - foreach (Uri allowListUrl in uniqueAllowListUrls) - tasks.Add(DownloadListUrlAsync(allowListUrl, true, false, false)); - - foreach (Uri blockListUrl in uniqueBlockListUrls) - tasks.Add(DownloadListUrlAsync(blockListUrl, false, false, false)); - - foreach (Uri regexAllowListUrl in uniqueRegexAllowListUrls) - tasks.Add(DownloadListUrlAsync(regexAllowListUrl, true, true, false)); - - foreach (Uri regexBlockListUrl in uniqueRegexBlockListUrls) - tasks.Add(DownloadListUrlAsync(regexBlockListUrl, false, true, false)); - - foreach (Uri adblockListUrl in uniqueAdblockListUrls) - tasks.Add(DownloadListUrlAsync(adblockListUrl, false, false, true)); - - await Task.WhenAll(tasks); - - bool downloaded = (downloadedAllowListUrls.Count > 0) || (downloadedBlockListUrls.Count > 0) || (downloadedRegexAllowListUrls.Count > 0) || (downloadedRegexBlockListUrls.Count > 0) || (downloadedAdblockListUrls.Count > 0); - if (downloaded) - LoadZones(downloadedAllowListUrls, downloadedBlockListUrls, downloadedRegexAllowListUrls, downloadedRegexBlockListUrls, downloadedAdblockListUrls); - - return downloaded || notModified; - } - - private static string PopWord(ref string line) - { - if (line.Length == 0) - return line; - - line = line.TrimStart(' ', '\t'); - - int i = line.IndexOfAny(new char[] { ' ', '\t' }); - string word; - - if (i < 0) - { - word = line; - line = ""; - } - else - { - word = line.Substring(0, i); - line = line.Substring(i + 1); - } - - return word; - } - - private Queue ReadListFile(Uri listUrl, bool isAllowList) - { - Queue domains = new Queue(); - - try - { - _dnsServer.WriteLog("Advanced Blocking app is reading " + (isAllowList ? "allow" : "block") + " list from: " + listUrl.AbsoluteUri); - - using (FileStream fS = new FileStream(GetListFilePath(listUrl), FileMode.Open, FileAccess.Read)) - { - //parse hosts file and populate block zone - StreamReader sR = new StreamReader(fS, true); - string line; - string firstWord; - string secondWord; - string hostname; - - while (true) - { - line = sR.ReadLine(); - if (line == null) - break; //eof - - line = line.TrimStart(' ', '\t'); - - if (line.Length == 0) - continue; //skip empty line - - if (line.StartsWith("#")) - continue; //skip comment line - - firstWord = PopWord(ref line); - - if (line.Length == 0) - { - hostname = firstWord; - } - else - { - secondWord = PopWord(ref line); - - if (secondWord.Length == 0) - hostname = firstWord; - else - hostname = secondWord; - } - - hostname = hostname.Trim('.').ToLower(); - - switch (hostname) - { - case "": - case "localhost": - case "localhost.localdomain": - case "local": - case "broadcasthost": - case "ip6-localhost": - case "ip6-loopback": - case "ip6-localnet": - case "ip6-mcastprefix": - case "ip6-allnodes": - case "ip6-allrouters": - case "ip6-allhosts": - continue; //skip these hostnames - } - - if (!DnsClient.IsDomainNameValid(hostname)) - continue; - - if (IPAddress.TryParse(hostname, out _)) - continue; //skip line when hostname is IP address - - domains.Enqueue(hostname); - } - } - - _dnsServer.WriteLog("Advanced Blocking app read " + (isAllowList ? "allow" : "block") + " list file (" + domains.Count + " domains) from: " + listUrl.AbsoluteUri); - } - catch (Exception ex) - { - _dnsServer.WriteLog("Advanced Blocking app failed to read " + (isAllowList ? "allow" : "block") + " list from: " + listUrl.AbsoluteUri + "\r\n" + ex.ToString()); - } - - return domains; - } - - private Queue ReadRegexListFile(Uri listUrl, bool isAllowList) - { - Queue regices = new Queue(); - - try - { - _dnsServer.WriteLog("Advanced Blocking app is reading regex " + (isAllowList ? "allow" : "block") + " list from: " + listUrl.AbsoluteUri); - - using (FileStream fS = new FileStream(GetListFilePath(listUrl), FileMode.Open, FileAccess.Read)) - { - //parse hosts file and populate block zone - StreamReader sR = new StreamReader(fS, true); - string line; - - while (true) - { - line = sR.ReadLine(); - if (line == null) - break; //eof - - line = line.TrimStart(' ', '\t'); - - if (line.Length == 0) - continue; //skip empty line - - if (line.StartsWith("#")) - continue; //skip comment line - - regices.Enqueue(line); - } - } - - _dnsServer.WriteLog("Advanced Blocking app read regex " + (isAllowList ? "allow" : "block") + " list file (" + regices.Count + " regex patterns) from: " + listUrl.AbsoluteUri); - } - catch (Exception ex) - { - _dnsServer.WriteLog("Advanced Blocking app failed to read regex " + (isAllowList ? "allow" : "block") + " list from: " + listUrl.AbsoluteUri + "\r\n" + ex.ToString()); - } - - return regices; - } - - private void ReadAdblockListFile(Uri listUrl, out Queue allowedDomains, out Queue blockedDomains) - { - allowedDomains = new Queue(); - blockedDomains = new Queue(); - - try - { - _dnsServer.WriteLog("Advanced Blocking app is reading adblock list from: " + listUrl.AbsoluteUri); - - using (FileStream fS = new FileStream(GetListFilePath(listUrl), FileMode.Open, FileAccess.Read)) - { - //parse hosts file and populate block zone - StreamReader sR = new StreamReader(fS, true); - string line; - - while (true) - { - line = sR.ReadLine(); - if (line == null) - break; //eof - - line = line.TrimStart(' ', '\t'); - - if (line.Length == 0) - continue; //skip empty line - - if (line.StartsWith("!")) - continue; //skip comment line - - if (line.StartsWith("||")) - { - int i = line.IndexOf('^'); - if (i > -1) - { - string domain = line.Substring(2, i - 2); - string options = line.Substring(i + 1); - - if (((options.Length == 0) || (options.StartsWith("$") && (options.Contains("doc") || options.Contains("all")))) && DnsClient.IsDomainNameValid(domain)) - blockedDomains.Enqueue(domain); - } - else - { - string domain = line.Substring(2); - - if (DnsClient.IsDomainNameValid(domain)) - blockedDomains.Enqueue(domain); - } - } - else if (line.StartsWith("@@||")) - { - int i = line.IndexOf('^'); - if (i > -1) - { - string domain = line.Substring(4, i - 4); - string options = line.Substring(i + 1); - - if (((options.Length == 0) || (options.StartsWith("$") && (options.Contains("doc") || options.Contains("all")))) && DnsClient.IsDomainNameValid(domain)) - blockedDomains.Enqueue(domain); - } - else - { - string domain = line.Substring(4); - - if (DnsClient.IsDomainNameValid(domain)) - allowedDomains.Enqueue(domain); - } - } - } - } - - _dnsServer.WriteLog("Advanced Blocking app read adblock list file (" + (allowedDomains.Count + blockedDomains.Count) + " domains) from: " + listUrl.AbsoluteUri); - } - catch (Exception ex) - { - _dnsServer.WriteLog("Advanced Blocking app failed to read adblock list from: " + listUrl.AbsoluteUri + "\r\n" + ex.ToString()); - } - } - - private IReadOnlyList GetUniqueAllowListUrls() - { - List allowListUrls = new List(); - - foreach (KeyValuePair group in _groups) - { - foreach (Uri allowListUrl in group.Value.AllowListUrls) - { - if (!allowListUrls.Contains(allowListUrl)) - allowListUrls.Add(allowListUrl); - } - } - - return allowListUrls; - } - - private IReadOnlyList GetUniqueBlockListUrls() - { - List blockListUrls = new List(); - - foreach (KeyValuePair group in _groups) - { - foreach (Uri blockListUrl in group.Value.BlockListUrls) - { - if (!blockListUrls.Contains(blockListUrl)) - blockListUrls.Add(blockListUrl); - } - } - - return blockListUrls; - } - - private IReadOnlyList GetUniqueRegexAllowListUrls() - { - List regexAllowListUrls = new List(); - - foreach (KeyValuePair group in _groups) - { - foreach (Uri regexAllowListUrl in group.Value.RegexAllowListUrls) - { - if (!regexAllowListUrls.Contains(regexAllowListUrl)) - regexAllowListUrls.Add(regexAllowListUrl); - } - } - - return regexAllowListUrls; - } - - private IReadOnlyList GetUniqueRegexBlockListUrls() - { - List regexBlockListUrls = new List(); - - foreach (KeyValuePair group in _groups) - { - foreach (Uri regexBlockListUrl in group.Value.RegexBlockListUrls) - { - if (!regexBlockListUrls.Contains(regexBlockListUrl)) - regexBlockListUrls.Add(regexBlockListUrl); - } - } - - return regexBlockListUrls; - } - - private IReadOnlyList GetUniqueAdblockListUrls() - { - List adblockListUrls = new List(); - - foreach (KeyValuePair group in _groups) - { - foreach (Uri adblockListUrl in group.Value.AdblockListUrls) - { - if (!adblockListUrls.Contains(adblockListUrl)) - adblockListUrls.Add(adblockListUrl); - } - } - - return adblockListUrls; - } - - private static IReadOnlyList GetAllUniqueListUrls(IReadOnlyDictionary groups) - { - List listUrls = new List(); - - foreach (KeyValuePair group in groups) - { - foreach (Uri allowListUrl in group.Key.AllowListUrls) - { - if (!listUrls.Contains(allowListUrl)) - listUrls.Add(allowListUrl); - } - - foreach (Uri blockListUrl in group.Key.BlockListUrls) - { - if (!listUrls.Contains(blockListUrl)) - listUrls.Add(blockListUrl); - } - - foreach (Uri regexAllowListUrl in group.Key.RegexAllowListUrls) - { - if (!listUrls.Contains(regexAllowListUrl)) - listUrls.Add(regexAllowListUrl); - } - - foreach (Uri regexBlockListUrl in group.Key.RegexBlockListUrls) - { - if (!listUrls.Contains(regexBlockListUrl)) - listUrls.Add(regexBlockListUrl); - } - - foreach (Uri adblockListUrl in group.Key.AdblockListUrls) - { - if (!listUrls.Contains(adblockListUrl)) - listUrls.Add(adblockListUrl); - } - } - - return listUrls; - } - - private void LoadZones(List updatedAllowListUrls, List updatedBlockListUrls, List updatedRegexAllowListUrls, List updatedRegexBlockListUrls, List updatedAdblockListUrls) - { - Dictionary> allowCache = new Dictionary>(); - Dictionary> blockCache = new Dictionary>(); - - foreach (KeyValuePair group in _groups) - { - bool loadAllowList = ListContainsAnyItem(group.Value.AllowListUrls, updatedAllowListUrls); - bool loadBlockList = ListContainsAnyItem(group.Value.BlockListUrls, updatedBlockListUrls); - bool loadRegexAllowList = ListContainsAnyItem(group.Value.RegexAllowListUrls, updatedRegexAllowListUrls); - bool loadRegexBlockList = ListContainsAnyItem(group.Value.RegexBlockListUrls, updatedRegexBlockListUrls); - bool loadAdblockList = ListContainsAnyItem(group.Value.AdblockListUrls, updatedAdblockListUrls); - - LoadListZones(allowCache, blockCache, group.Value, loadAllowList, loadBlockList, loadRegexAllowList, loadRegexBlockList, loadAdblockList); - } - } - - private void LoadListZones(Dictionary> allowCache, Dictionary> blockCache, Group group, bool loadAllowList, bool loadBlockList, bool loadRegexAllowList, bool loadRegexBlockList, bool loadAdblockList) - { - if (loadAdblockList) - { - loadAllowList = true; - loadBlockList = true; - } - - Dictionary> allAllowListQueues = new Dictionary>(); - Dictionary> allBlockListQueues = new Dictionary>(); - Dictionary> allRegexAllowListQueues = new Dictionary>(); - Dictionary> allRegexBlockListQueues = new Dictionary>(); - - if (loadAllowList) - { - //read all allow lists in a queue - foreach (Uri allowListUrl in group.AllowListUrls) - { - if (allAllowListQueues.ContainsKey(allowListUrl)) - continue; - - if (!allowCache.TryGetValue(allowListUrl, out Queue allowListQueue)) - { - allowListQueue = ReadListFile(allowListUrl, true); - allowCache.Add(allowListUrl, allowListQueue); - } - - allAllowListQueues.Add(allowListUrl, allowListQueue); - } - } - - if (loadBlockList) - { - //read all block lists in a queue - foreach (Uri blockListUrl in group.BlockListUrls) - { - if (allBlockListQueues.ContainsKey(blockListUrl)) - continue; - - if (!blockCache.TryGetValue(blockListUrl, out Queue blockListQueue)) - { - blockListQueue = ReadListFile(blockListUrl, false); - blockCache.Add(blockListUrl, blockListQueue); - } - - allBlockListQueues.Add(blockListUrl, blockListQueue); - } - } - - if (loadAdblockList) - { - //read all adblock lists in queue - foreach (Uri adblockListUrl in group.AdblockListUrls) - { - if (!allowCache.TryGetValue(adblockListUrl, out Queue allowListQueue) & !blockCache.TryGetValue(adblockListUrl, out Queue blockListQueue)) - { - ReadAdblockListFile(adblockListUrl, out allowListQueue, out blockListQueue); - - allowCache.Add(adblockListUrl, allowListQueue); - blockCache.Add(adblockListUrl, blockListQueue); - } - - allAllowListQueues.Add(adblockListUrl, allowListQueue); - allBlockListQueues.Add(adblockListUrl, blockListQueue); - } - } - - if (loadRegexAllowList) - { - //read all allow lists in a queue - foreach (Uri regexAllowListUrl in group.RegexAllowListUrls) - { - if (allRegexAllowListQueues.ContainsKey(regexAllowListUrl)) - continue; - - if (!allowCache.TryGetValue(regexAllowListUrl, out Queue regexAllowListQueue)) - { - regexAllowListQueue = ReadRegexListFile(regexAllowListUrl, true); - allowCache.Add(regexAllowListUrl, regexAllowListQueue); - } - - allRegexAllowListQueues.Add(regexAllowListUrl, regexAllowListQueue); - } - } - - if (loadRegexBlockList) - { - //read all regex block lists in a queue - foreach (Uri regexBlockListUrl in group.RegexBlockListUrls) - { - if (allRegexBlockListQueues.ContainsKey(regexBlockListUrl)) - continue; - - if (!blockCache.TryGetValue(regexBlockListUrl, out Queue regexBlockListQueue)) - { - regexBlockListQueue = ReadRegexListFile(regexBlockListUrl, false); - blockCache.Add(regexBlockListUrl, regexBlockListQueue); - } - - allRegexBlockListQueues.Add(regexBlockListUrl, regexBlockListQueue); - } - } - - //load block list zone - if (loadAllowList) - group.LoadAllowListZone(allAllowListQueues); - - if (loadBlockList) - group.LoadBlockListZone(allBlockListQueues); - - //load regex block list zone - if (loadRegexAllowList) - group.LoadRegexAllowListZone(allRegexAllowListQueues); - - if (loadRegexBlockList) - group.LoadRegexBlockListZone(allRegexBlockListQueues); - - _dnsServer.WriteLog("Advanced Blocking app loaded all zones successfully for group: " + group.Name); - } - - private static bool ListsEquals(IReadOnlyList list1, IReadOnlyList list2) - { - if (list1.Count != list2.Count) - return false; - - foreach (T item in list1) - { - if (!list2.Contains(item)) - return false; - } - - return true; - } - - private static bool ListContainsAnyItem(IReadOnlyList list, IReadOnlyList items) - { - foreach (T item in list) - { - if (items.Contains(item)) + bool downloaded = await updateTask; + if (downloaded) return true; } @@ -797,6 +141,104 @@ namespace AdvancedBlocking return null; } + private static bool IsZoneFound(IReadOnlyDictionary domains, string domain, out string foundZone) + { + do + { + if (domains.TryGetValue(domain, out _)) + { + foundZone = domain; + return true; + } + + domain = GetParentZone(domain); + } + while (domain is not null); + + foundZone = null; + return false; + } + + private static bool IsZoneFound(IReadOnlyDictionary listZones, string domain, out string foundZone, out Uri listUri) + { + foreach (KeyValuePair listZone in listZones) + { + if (listZone.Value.IsZoneFound(domain, out foundZone)) + { + listUri = listZone.Key; + return true; + } + } + + foundZone = null; + listUri = null; + return false; + } + + private static bool IsZoneAllowed(IReadOnlyDictionary listZones, string domain, out string foundZone, out Uri listUri) + { + foreach (KeyValuePair listZone in listZones) + { + if (listZone.Value.IsZoneAllowed(domain, out foundZone)) + { + listUri = listZone.Key; + return true; + } + } + + foundZone = null; + listUri = null; + return false; + } + + private static bool IsZoneBlocked(IReadOnlyDictionary listZones, string domain, out string foundZone, out Uri listUri) + { + foreach (KeyValuePair listZone in listZones) + { + if (listZone.Value.IsZoneBlocked(domain, out foundZone)) + { + listUri = listZone.Key; + return true; + } + } + + foundZone = null; + listUri = null; + return false; + } + + private static bool IsMatchFound(IReadOnlyList regices, string domain, out string matchingPattern) + { + foreach (Regex regex in regices) + { + if (regex.IsMatch(domain)) + { + //found pattern + matchingPattern = regex.ToString(); + return true; + } + } + + matchingPattern = null; + return false; + } + + private static bool IsMatchFound(IReadOnlyDictionary regexListZones, string domain, out string matchingPattern, out Uri listUri) + { + foreach (KeyValuePair regexListZone in regexListZones) + { + if (regexListZone.Value.IsMatchFound(domain, out matchingPattern)) + { + listUri = regexListZone.Key; + return true; + } + } + + matchingPattern = null; + listUri = null; + return false; + } + #endregion #region public @@ -804,9 +246,8 @@ namespace AdvancedBlocking public Task InitializeAsync(IDnsServer dnsServer, string config) { _dnsServer = dnsServer; - _localCacheFolder = Path.Combine(_dnsServer.ApplicationFolder, "blocklists"); - Directory.CreateDirectory(_localCacheFolder); + Directory.CreateDirectory(Path.Combine(_dnsServer.ApplicationFolder, "blocklists")); _soaRecord = new DnsSOARecordData(_dnsServer.ServerDomain, "hostadmin@" + _dnsServer.ServerDomain, 1, 14400, 3600, 604800, 60); _nsRecord = new DnsNSRecordData(_dnsServer.ServerDomain); @@ -831,123 +272,157 @@ namespace AdvancedBlocking _networkGroupMap = networkGroupMap; } - bool cachedListFileMissing = false; - { - const int LOAD_ALLOW_LIST_ZONE = 1; - const int LOAD_BLOCK_LIST_ZONE = 2; - const int LOAD_REGEX_ALLOW_LIST_ZONE = 4; - const int LOAD_REGEX_BLOCK_LIST_ZONE = 8; - const int LOAD_ADBLOCK_LIST_ZONE = 16; - - Dictionary updatedGroups = new Dictionary(); Dictionary groups = new Dictionary(); + Dictionary allAllowListZones = new Dictionary(0); + Dictionary allBlockListZones = new Dictionary(0); + + Dictionary allRegexAllowListZones = new Dictionary(0); + Dictionary allRegexBlockListZones = new Dictionary(0); + + Dictionary allAdBlockListZones = new Dictionary(0); + foreach (dynamic jsonGroup in jsonConfig.groups) { Group group = new Group(this, jsonGroup); + if (!groups.TryAdd(group.Name, group)) + continue; - if ((_groups is not null) && _groups.TryGetValue(group.Name, out Group existingGroup)) + foreach (Uri allowListUrl in group.AllowListUrls) { - int loadFlags = 0; - - if (!ListsEquals(group.AllowListUrls, existingGroup.AllowListUrls)) - loadFlags |= LOAD_ALLOW_LIST_ZONE; - - if (!ListsEquals(group.BlockListUrls, existingGroup.BlockListUrls)) - loadFlags |= LOAD_BLOCK_LIST_ZONE; - - if (!ListsEquals(group.RegexAllowListUrls, existingGroup.RegexAllowListUrls)) - loadFlags |= LOAD_REGEX_ALLOW_LIST_ZONE; - - if (!ListsEquals(group.RegexBlockListUrls, existingGroup.RegexBlockListUrls)) - loadFlags |= LOAD_REGEX_BLOCK_LIST_ZONE; - - if (!ListsEquals(group.AdblockListUrls, existingGroup.AdblockListUrls)) - loadFlags |= LOAD_ADBLOCK_LIST_ZONE; - - if (loadFlags > 0) - updatedGroups.Add(existingGroup, loadFlags); - - existingGroup.EnableBlocking = group.EnableBlocking; - existingGroup.AllowTxtBlockingReport = group.AllowTxtBlockingReport; - existingGroup.BlockAsNxDomain = group.BlockAsNxDomain; - existingGroup.ARecords = group.ARecords; - existingGroup.AAAARecords = group.AAAARecords; - - existingGroup.Allowed = group.Allowed; - existingGroup.Blocked = group.Blocked; - existingGroup.AllowListUrls = group.AllowListUrls; - existingGroup.BlockListUrls = group.BlockListUrls; - - existingGroup.AllowedRegex = group.AllowedRegex; - existingGroup.BlockedRegex = group.BlockedRegex; - existingGroup.RegexAllowListUrls = group.RegexAllowListUrls; - existingGroup.RegexBlockListUrls = group.RegexBlockListUrls; - - existingGroup.AdblockListUrls = group.AdblockListUrls; - - groups.TryAdd(existingGroup.Name, existingGroup); + if (!allAllowListZones.ContainsKey(allowListUrl)) + { + if (_allAllowListZones.TryGetValue(allowListUrl, out BlockList allowList)) + allAllowListZones.Add(allowListUrl, allowList); + else + allAllowListZones.Add(allowListUrl, new BlockList(_dnsServer, allowListUrl, true)); + } } - else + + foreach (Uri blockListUrl in group.BlockListUrls) { - updatedGroups.Add(group, LOAD_ALLOW_LIST_ZONE | LOAD_BLOCK_LIST_ZONE | LOAD_REGEX_ALLOW_LIST_ZONE | LOAD_REGEX_BLOCK_LIST_ZONE | LOAD_ADBLOCK_LIST_ZONE); - groups.TryAdd(group.Name, group); + if (!allBlockListZones.ContainsKey(blockListUrl)) + { + if (_allBlockListZones.TryGetValue(blockListUrl, out BlockList blockList)) + allBlockListZones.Add(blockListUrl, blockList); + else + allBlockListZones.Add(blockListUrl, new BlockList(_dnsServer, blockListUrl, false)); + } + } + + foreach (Uri regexAllowListUrl in group.RegexAllowListUrls) + { + if (!allRegexAllowListZones.ContainsKey(regexAllowListUrl)) + { + if (_allRegexAllowListZones.TryGetValue(regexAllowListUrl, out RegexList regexAllowList)) + allRegexAllowListZones.Add(regexAllowListUrl, regexAllowList); + else + allRegexAllowListZones.Add(regexAllowListUrl, new RegexList(_dnsServer, regexAllowListUrl, true)); + } + } + + foreach (Uri regexBlockListUrl in group.RegexBlockListUrls) + { + if (!allRegexBlockListZones.ContainsKey(regexBlockListUrl)) + { + if (_allRegexBlockListZones.TryGetValue(regexBlockListUrl, out RegexList regexBlockList)) + allRegexBlockListZones.Add(regexBlockListUrl, regexBlockList); + else + allRegexBlockListZones.Add(regexBlockListUrl, new RegexList(_dnsServer, regexBlockListUrl, false)); + } + } + + foreach (Uri adblockListUrl in group.AdblockListUrls) + { + if (!allAdBlockListZones.ContainsKey(adblockListUrl)) + { + if (_allAdBlockListZones.TryGetValue(adblockListUrl, out AdBlockList adBlockList)) + allAdBlockListZones.Add(adblockListUrl, adBlockList); + else + allAdBlockListZones.Add(adblockListUrl, new AdBlockList(_dnsServer, adblockListUrl)); + } } } _groups = groups; - if (updatedGroups.Count > 0) + _allAllowListZones = allAllowListZones; + _allBlockListZones = allBlockListZones; + + _allRegexAllowListZones = allRegexAllowListZones; + _allRegexBlockListZones = allRegexBlockListZones; + + _allAdBlockListZones = allAdBlockListZones; + } + + foreach (KeyValuePair group in _groups) + { + group.Value.LoadListZones(); + _dnsServer.WriteLog("Advanced Blocking app loaded all zones successfully for group: " + group.Key); + } + + Task.Run(async delegate () + { + List loadTasks = new List(); + + foreach (KeyValuePair allAllowListZone in _allAllowListZones) + loadTasks.Add(allAllowListZone.Value.LoadAsync()); + + foreach (KeyValuePair allBlockListZone in _allBlockListZones) + loadTasks.Add(allBlockListZone.Value.LoadAsync()); + + foreach (KeyValuePair allRegexAllowListZone in _allRegexAllowListZones) + loadTasks.Add(allRegexAllowListZone.Value.LoadAsync()); + + foreach (KeyValuePair allRegexBlockListZone in _allRegexBlockListZones) + loadTasks.Add(allRegexBlockListZone.Value.LoadAsync()); + + foreach (KeyValuePair allAdBlockListZone in _allAdBlockListZones) + loadTasks.Add(allAdBlockListZone.Value.LoadAsync()); + + await Task.WhenAll(loadTasks); + + if (_blockListUrlUpdateTimer is null) { - foreach (Uri listUrl in GetAllUniqueListUrls(updatedGroups)) + DateTime latest = DateTime.MinValue; + + foreach (KeyValuePair allAllowListZone in _allAllowListZones) { - if (!File.Exists(GetListFilePath(listUrl))) - { - cachedListFileMissing = true; - break; - } + if (allAllowListZone.Value.LastModified > latest) + latest = allAllowListZone.Value.LastModified; } - if (!cachedListFileMissing) + foreach (KeyValuePair allBlockListZone in _allBlockListZones) { - Task.Run(delegate () - { - Dictionary> allowCache = new Dictionary>(); - Dictionary> blockCache = new Dictionary>(); - - foreach (KeyValuePair group in updatedGroups) - { - bool loadAllowList = (group.Value & LOAD_ALLOW_LIST_ZONE) > 0; - bool loadBlockList = (group.Value & LOAD_BLOCK_LIST_ZONE) > 0; - bool loadRegexAllowList = (group.Value & LOAD_REGEX_ALLOW_LIST_ZONE) > 0; - bool loadRegexBlockList = (group.Value & LOAD_REGEX_BLOCK_LIST_ZONE) > 0; - bool loadAdblockList = (group.Value & LOAD_ADBLOCK_LIST_ZONE) > 0; - - LoadListZones(allowCache, blockCache, group.Key, loadAllowList, loadBlockList, loadRegexAllowList, loadRegexBlockList, loadAdblockList); - } - }); + if (allBlockListZone.Value.LastModified > latest) + latest = allBlockListZone.Value.LastModified; } - } - } - if (_blockListUrlUpdateTimer is null) - { - if (!cachedListFileMissing) - FindAndSetBlockListUrlLastUpdatedOn(); + foreach (KeyValuePair allRegexAllowListZone in _allRegexAllowListZones) + { + if (allRegexAllowListZone.Value.LastModified > latest) + latest = allRegexAllowListZone.Value.LastModified; + } - _blockListUrlUpdateTimer = new Timer(BlockListUrlUpdateTimerCallbackAsync, null, Timeout.Infinite, Timeout.Infinite); - _blockListUrlUpdateTimer.Change(BLOCK_LIST_UPDATE_TIMER_INITIAL_INTERVAL, BLOCK_LIST_UPDATE_TIMER_PERIODIC_INTERVAL); - } - else - { - if (cachedListFileMissing) - { - //force update - _blockListUrlLastUpdatedOn = DateTime.MinValue; - _blockListUrlUpdateTimer.Change(BLOCK_LIST_UPDATE_TIMER_INITIAL_INTERVAL, BLOCK_LIST_UPDATE_TIMER_PERIODIC_INTERVAL); + foreach (KeyValuePair allRegexBlockListZone in _allRegexBlockListZones) + { + if (allRegexBlockListZone.Value.LastModified > latest) + latest = allRegexBlockListZone.Value.LastModified; + } + + foreach (KeyValuePair allAdBlockListZone in _allAdBlockListZones) + { + if (allAdBlockListZone.Value.LastModified > latest) + latest = allAdBlockListZone.Value.LastModified; + } + + _blockListUrlLastUpdatedOn = latest; + + _blockListUrlUpdateTimer = new Timer(BlockListUrlUpdateTimerCallbackAsync, null, Timeout.Infinite, Timeout.Infinite); + _blockListUrlUpdateTimer.Change(BLOCK_LIST_UPDATE_TIMER_INTERVAL, BLOCK_LIST_UPDATE_TIMER_INTERVAL); } - } + }); return Task.CompletedTask; } @@ -958,14 +433,15 @@ namespace AdvancedBlocking return Task.FromResult(null); IPAddress remoteIP = remoteEP.Address; + NetworkAddress network = null; string groupName = null; foreach (KeyValuePair entry in _networkGroupMap) { - if (entry.Key.Contains(remoteIP)) + if (entry.Key.Contains(remoteIP) && ((network is null) || (entry.Key.PrefixLength > network.PrefixLength))) { + network = entry.Key; groupName = entry.Value; - break; } } @@ -974,8 +450,7 @@ namespace AdvancedBlocking DnsQuestionRecord question = request.Question[0]; - IReadOnlyList blockListUrls = group.IsZoneBlocked(question.Name, out string blockedDomain, out string blockedRegex); - if (blockListUrls is null) + if (!group.IsZoneBlocked(question.Name, out string blockedDomain, out string blockedRegex, out Uri blockListUrl)) return Task.FromResult(null); if (group.AllowTxtBlockingReport && (question.Type == DnsResourceRecordType.TXT)) @@ -985,31 +460,17 @@ namespace AdvancedBlocking if (blockedRegex is null) { - if (blockListUrls.Count > 0) - { - answer = new DnsResourceRecord[blockListUrls.Count]; - - for (int i = 0; i < answer.Length; i++) - answer[i] = new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData("source=advanced-blocking-app; group=" + group.Name + "; blockListUrl=" + blockListUrls[i].AbsoluteUri + "; domain=" + blockedDomain)); - } + if (blockListUrl is not null) + answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData("source=advanced-blocking-app; group=" + group.Name + "; blockListUrl=" + blockListUrl.AbsoluteUri + "; domain=" + blockedDomain)) }; else - { answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData("source=advanced-blocking-app; group=" + group.Name + "; domain=" + blockedDomain)) }; - } } else { - if (blockListUrls.Count > 0) - { - answer = new DnsResourceRecord[blockListUrls.Count]; - - for (int i = 0; i < answer.Length; i++) - answer[i] = new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData("source=advanced-blocking-app; group=" + group.Name + "; regexBlockListUrl=" + blockListUrls[i].AbsoluteUri + "; regex=" + blockedRegex)); - } + if (blockListUrl is not null) + answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData("source=advanced-blocking-app; group=" + group.Name + "; regexBlockListUrl=" + blockListUrl.AbsoluteUri + "; regex=" + blockedRegex)) }; else - { answer = new DnsResourceRecord[] { new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, 60, new DnsTXTRecordData("source=advanced-blocking-app; group=" + group.Name + "; regex=" + blockedRegex)) }; - } } return Task.FromResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, isRecursionAllowed, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = DnsServerResponseType.Blocked }); @@ -1108,30 +569,32 @@ namespace AdvancedBlocking readonly App _app; readonly string _name; - bool _enableBlocking; - bool _allowTxtBlockingReport; - bool _blockAsNxDomain; + readonly bool _enableBlocking; + readonly bool _allowTxtBlockingReport; + readonly bool _blockAsNxDomain; - IReadOnlyCollection _aRecords; - IReadOnlyCollection _aaaaRecords; + readonly IReadOnlyCollection _aRecords; + readonly IReadOnlyCollection _aaaaRecords; - IReadOnlyDictionary _allowed; - IReadOnlyDictionary _blocked; - IReadOnlyList _allowListUrls; - IReadOnlyList _blockListUrls; + readonly IReadOnlyDictionary _allowed; + readonly IReadOnlyDictionary _blocked; + readonly IReadOnlyList _allowListUrls; + readonly IReadOnlyList _blockListUrls; - IReadOnlyList _allowedRegex; - IReadOnlyList _blockedRegex; - IReadOnlyList _regexAllowListUrls; - IReadOnlyList _regexBlockListUrls; + readonly IReadOnlyList _allowedRegex; + readonly IReadOnlyList _blockedRegex; + readonly IReadOnlyList _regexAllowListUrls; + readonly IReadOnlyList _regexBlockListUrls; - IReadOnlyList _adblockListUrls; + readonly IReadOnlyList _adblockListUrls; - IReadOnlyDictionary> _allowListZone = new Dictionary>(0); - IReadOnlyDictionary> _blockListZone = new Dictionary>(0); + IReadOnlyDictionary _allowListZones = new Dictionary(0); + IReadOnlyDictionary _blockListZones = new Dictionary(0); - IReadOnlyList _regexAllowListZone = Array.Empty(); - IReadOnlyList _regexBlockListZone = Array.Empty(); + IReadOnlyDictionary _regexAllowListZones = new Dictionary(0); + IReadOnlyDictionary _regexBlockListZones = new Dictionary(0); + + IReadOnlyDictionary _adBlockListZones = new Dictionary(0); #endregion @@ -1228,218 +691,105 @@ namespace AdvancedBlocking return urls; } - private static bool IsZoneFound(IReadOnlyDictionary domains, string domain, out string foundZone, out T foundValue) where T : class - { - do - { - if (domains.TryGetValue(domain, out T value)) - { - foundZone = domain; - foundValue = value; - return true; - } - - domain = GetParentZone(domain); - } - while (domain is not null); - - foundZone = null; - foundValue = null; - return false; - } - - private static bool IsMatchFound(IReadOnlyList regices, string domain, out string matchingPattern) - { - foreach (Regex regex in regices) - { - if (regex.IsMatch(domain)) - { - //found pattern - matchingPattern = regex.ToString(); - return true; - } - } - - matchingPattern = null; - return false; - } - - private static bool IsMatchFound(IReadOnlyList regices, string domain, out string matchingPattern, out IReadOnlyList blockListUrls) - { - foreach (RegexItem regex in regices) - { - if (regex.Regex.IsMatch(domain)) - { - //found pattern - matchingPattern = regex.Regex.ToString(); - blockListUrls = regex.BlockListUrls; - return true; - } - } - - matchingPattern = null; - blockListUrls = null; - return false; - } - - private static IReadOnlyDictionary> LoadListZone(IReadOnlyList listUrls, Dictionary> allListQueues) - { - //select lists - Dictionary> listQueues = new Dictionary>(listUrls.Count); - int totalDomains = 0; - - foreach (Uri listUrl in listUrls) - { - if (allListQueues.TryGetValue(listUrl, out Queue listQueue)) - { - totalDomains += listQueue.Count; - listQueues.Add(listUrl, listQueue); - } - } - - //load list zone - Dictionary> listZone = new Dictionary>(totalDomains); - - foreach (KeyValuePair> listQueue in listQueues) - { - Queue queue = listQueue.Value; - - while (queue.Count > 0) - { - string domain = queue.Dequeue(); - - if (!listZone.TryGetValue(domain, out List sourceListUrls)) - { - sourceListUrls = new List(2); - listZone.Add(domain, sourceListUrls); - } - - sourceListUrls.Add(listQueue.Key); - } - } - - return listZone; - } - - private IReadOnlyList LoadRegexListZone(IReadOnlyList regexListUrls, Dictionary> allRegexListQueues) - { - //select regex lists - Dictionary> regexListQueues = new Dictionary>(regexListUrls.Count); - int totalRegexPatterns = 0; - - foreach (Uri regexListUrl in regexListUrls) - { - if (allRegexListQueues.TryGetValue(regexListUrl, out Queue regexListQueue)) - { - totalRegexPatterns += regexListQueue.Count; - regexListQueues.Add(regexListUrl, regexListQueue); - } - } - - //load regex list patterns from queue - Dictionary> allRegexPatterns = new Dictionary>(totalRegexPatterns); - - foreach (KeyValuePair> regexListQueue in regexListQueues) - { - Queue queue = regexListQueue.Value; - - while (queue.Count > 0) - { - string regex = queue.Dequeue(); - - if (!allRegexPatterns.TryGetValue(regex, out List sourceListUrls)) - { - sourceListUrls = new List(2); - allRegexPatterns.Add(regex, sourceListUrls); - } - - sourceListUrls.Add(regexListQueue.Key); - } - } - - //load regex list zone - List regexListZone = new List(totalRegexPatterns); - - foreach (KeyValuePair> regexPattern in allRegexPatterns) - { - try - { - Regex regex = new Regex(regexPattern.Key, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); - - regexListZone.Add(new RegexItem(regex, regexPattern.Value)); - } - catch (RegexParseException ex) - { - _app._dnsServer.WriteLog(ex); - } - } - - return regexListZone; - } - #endregion #region public - public void LoadAllowListZone(Dictionary> allAllowListQueues) + public void LoadListZones() { - List listUrls = new List(); + { + Dictionary allowListZones = new Dictionary(_allowListUrls.Count); - listUrls.AddRange(_allowListUrls); - listUrls.AddRange(_adblockListUrls); + foreach (Uri listUrl in _allowListUrls) + { + if (_app._allAllowListZones.TryGetValue(listUrl, out BlockList allowListZone)) + allowListZones.Add(listUrl, allowListZone); + } - _allowListZone = LoadListZone(listUrls, allAllowListQueues); + _allowListZones = allowListZones; + } + + { + Dictionary blockListZones = new Dictionary(_blockListUrls.Count); + + foreach (Uri listUrl in _blockListUrls) + { + if (_app._allBlockListZones.TryGetValue(listUrl, out BlockList blockListZone)) + blockListZones.Add(listUrl, blockListZone); + } + + _blockListZones = blockListZones; + } + + { + Dictionary regexAllowListZones = new Dictionary(_regexAllowListUrls.Count); + + foreach (Uri listUrl in _regexAllowListUrls) + { + if (_app._allRegexAllowListZones.TryGetValue(listUrl, out RegexList regexAllowListZone)) + regexAllowListZones.Add(listUrl, regexAllowListZone); + } + + _regexAllowListZones = regexAllowListZones; + } + + { + Dictionary regexBlockListZones = new Dictionary(_regexBlockListUrls.Count); + + foreach (Uri listUrl in _regexBlockListUrls) + { + if (_app._allRegexBlockListZones.TryGetValue(listUrl, out RegexList regexBlockListZone)) + regexBlockListZones.Add(listUrl, regexBlockListZone); + } + + _regexBlockListZones = regexBlockListZones; + } + + { + Dictionary adBlockListZones = new Dictionary(_adblockListUrls.Count); + + foreach (Uri listUrl in _adblockListUrls) + { + if (_app._allAdBlockListZones.TryGetValue(listUrl, out AdBlockList adBlockListZone)) + adBlockListZones.Add(listUrl, adBlockListZone); + } + + _adBlockListZones = adBlockListZones; + } } - public void LoadBlockListZone(Dictionary> allBlockListQueues) - { - List listUrls = new List(); - - listUrls.AddRange(_blockListUrls); - listUrls.AddRange(_adblockListUrls); - - _blockListZone = LoadListZone(listUrls, allBlockListQueues); - } - - public void LoadRegexAllowListZone(Dictionary> allRegexAllowListQueues) - { - _regexAllowListZone = LoadRegexListZone(_regexAllowListUrls, allRegexAllowListQueues); - } - - public void LoadRegexBlockListZone(Dictionary> allRegexBlockListQueues) - { - _regexBlockListZone = LoadRegexListZone(_regexBlockListUrls, allRegexBlockListQueues); - } - - public IReadOnlyList IsZoneBlocked(string domain, out string blockedDomain, out string blockedRegex) + public bool IsZoneBlocked(string domain, out string blockedDomain, out string blockedRegex, out Uri listUrl) { domain = domain.ToLower(); - //allowed, allow list zone, allowedRegex, regex allow list zone - if (IsZoneFound(_allowed, domain, out _, out _) || IsZoneFound(_allowListZone, domain, out _, out _) || IsMatchFound(_allowedRegex, domain, out _) || IsMatchFound(_regexAllowListZone, domain, out _, out _)) + //allowed, allow list zone, allowedRegex, regex allow list zone, adblock list zone + if (IsZoneFound(_allowed, domain, out _) || IsZoneFound(_allowListZones, domain, out _, out _) || IsMatchFound(_allowedRegex, domain, out _) || IsMatchFound(_regexAllowListZones, domain, out _, out _) || IsZoneAllowed(_adBlockListZones, domain, out _, out _)) { //found zone allowed blockedDomain = null; blockedRegex = null; - return null; + listUrl = null; + return false; } //blocked - if (IsZoneFound(_blocked, domain, out string foundZone1, out _)) + if (IsZoneFound(_blocked, domain, out string foundZone1)) { //found zone blocked blockedDomain = foundZone1; blockedRegex = null; - return Array.Empty(); + listUrl = null; + return true; } //block list zone - if (IsZoneFound(_blockListZone, domain, out string foundZone2, out List blockListUrls1)) + if (IsZoneFound(_blockListZones, domain, out string foundZone2, out Uri blockListUrl1)) { //found zone blocked blockedDomain = foundZone2; blockedRegex = null; - return blockListUrls1; + listUrl = blockListUrl1; + return true; } //blockedRegex @@ -1448,21 +798,34 @@ namespace AdvancedBlocking //found pattern blocked blockedDomain = null; blockedRegex = blockedPattern1; - return Array.Empty(); + listUrl = null; + return true; } //regex block list zone - if (IsMatchFound(_regexBlockListZone, domain, out string blockedPattern2, out IReadOnlyList blockListUrls2)) + if (IsMatchFound(_regexBlockListZones, domain, out string blockedPattern2, out Uri blockListUrl2)) { //found pattern blocked blockedDomain = null; blockedRegex = blockedPattern2; - return blockListUrls2; + listUrl = blockListUrl2; + return true; + } + + //adblock list zone + if (App.IsZoneBlocked(_adBlockListZones, domain, out string foundZone3, out Uri blockListUrl3)) + { + //found zone blocked + blockedDomain = foundZone3; + blockedRegex = null; + listUrl = blockListUrl3; + return true; } blockedDomain = null; blockedRegex = null; - return null; + listUrl = null; + return false; } #endregion @@ -1473,118 +836,583 @@ namespace AdvancedBlocking { get { return _name; } } public bool EnableBlocking - { - get { return _enableBlocking; } - set { _enableBlocking = value; } - } + { get { return _enableBlocking; } } public bool AllowTxtBlockingReport - { - get { return _allowTxtBlockingReport; } - set { _allowTxtBlockingReport = value; } - } + { get { return _allowTxtBlockingReport; } } public bool BlockAsNxDomain - { - get { return _blockAsNxDomain; } - set { _blockAsNxDomain = value; } - } + { get { return _blockAsNxDomain; } } public IReadOnlyCollection ARecords - { - get { return _aRecords; } - set { _aRecords = value; } - } + { get { return _aRecords; } } public IReadOnlyCollection AAAARecords - { - get { return _aaaaRecords; } - set { _aaaaRecords = value; } - } - - public IReadOnlyDictionary Allowed - { - get { return _allowed; } - set { _allowed = value; } - } - - public IReadOnlyDictionary Blocked - { - get { return _blocked; } - set { _blocked = value; } - } + { get { return _aaaaRecords; } } public IReadOnlyList AllowListUrls - { - get { return _allowListUrls; } - set { _allowListUrls = value; } - } + { get { return _allowListUrls; } } public IReadOnlyList BlockListUrls - { - get { return _blockListUrls; } - set { _blockListUrls = value; } - } - - public IReadOnlyList AllowedRegex - { - get { return _allowedRegex; } - set { _allowedRegex = value; } - } - - public IReadOnlyList BlockedRegex - { - get { return _blockedRegex; } - set { _blockedRegex = value; } - } + { get { return _blockListUrls; } } public IReadOnlyList RegexBlockListUrls - { - get { return _regexBlockListUrls; } - set { _regexBlockListUrls = value; } - } + { get { return _regexBlockListUrls; } } public IReadOnlyList RegexAllowListUrls - { - get { return _regexAllowListUrls; } - set { _regexAllowListUrls = value; } - } + { get { return _regexAllowListUrls; } } public IReadOnlyList AdblockListUrls - { - get { return _adblockListUrls; } - set { _adblockListUrls = value; } - } + { get { return _adblockListUrls; } } #endregion } - class RegexItem + abstract class ListBase { #region variables - readonly Regex _regex; - readonly IReadOnlyList _blockListUrls; + protected readonly IDnsServer _dnsServer; + protected readonly Uri _listUrl; + protected readonly bool _isAllowList; + protected readonly bool _isRegexList; + protected readonly bool _isAdblockList; + + protected readonly string _listFilePath; + bool _listZoneLoaded; + DateTime _lastModified; + + volatile bool _isLoading; #endregion #region constructor - public RegexItem(Regex regex, IReadOnlyList blockListUrls) + public ListBase(IDnsServer dnsServer, Uri listUrl, bool isAllowList, bool isRegexList, bool isAdblockList) { - _regex = regex; - _blockListUrls = blockListUrls; + _dnsServer = dnsServer; + _listUrl = listUrl; + _isAllowList = isAllowList; + _isRegexList = isRegexList; + _isAdblockList = isAdblockList; + + using (HashAlgorithm hash = SHA256.Create()) + { + _listFilePath = Path.Combine(Path.Combine(_dnsServer.ApplicationFolder, "blocklists"), Convert.ToHexString(hash.ComputeHash(Encoding.UTF8.GetBytes(_listUrl.AbsoluteUri))).ToLower()); + } + } + + #endregion + + #region private + + private async Task DownloadListFileAsync() + { + try + { + _dnsServer.WriteLog("Advanced Blocking app is downloading " + (_isAdblockList ? "adblock" : (_isRegexList ? "regex " : "") + (_isAllowList ? "allow" : "block")) + " list: " + _listUrl.AbsoluteUri); + + SocketsHttpHandler handler = new SocketsHttpHandler(); + handler.Proxy = _dnsServer.Proxy; + handler.UseProxy = _dnsServer.Proxy is not null; + handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + + using (HttpClient http = new HttpClient(handler)) + { + if (File.Exists(_listFilePath)) + http.DefaultRequestHeaders.IfModifiedSince = File.GetLastWriteTimeUtc(_listFilePath); + + HttpResponseMessage httpResponse = await http.GetAsync(_listUrl); + switch (httpResponse.StatusCode) + { + case HttpStatusCode.OK: + string listDownloadFilePath = _listFilePath + ".downloading"; + + using (FileStream fS = new FileStream(listDownloadFilePath, FileMode.Create, FileAccess.Write)) + { + using (Stream httpStream = await httpResponse.Content.ReadAsStreamAsync()) + { + await httpStream.CopyToAsync(fS); + } + } + + File.Move(listDownloadFilePath, _listFilePath, true); + + if (httpResponse.Content.Headers.LastModified is null) + { + _lastModified = DateTime.UtcNow; + } + else + { + _lastModified = httpResponse.Content.Headers.LastModified.Value.UtcDateTime; + File.SetLastWriteTimeUtc(_listFilePath, _lastModified); + } + + _dnsServer.WriteLog("Advanced Blocking app successfully downloaded " + (_isAdblockList ? "adblock" : (_isRegexList ? "regex " : "") + (_isAllowList ? "allow" : "block")) + " list (" + WebUtilities.GetFormattedSize(new FileInfo(_listFilePath).Length) + "): " + _listUrl.AbsoluteUri); + return true; + + case HttpStatusCode.NotModified: + _dnsServer.WriteLog("Advanced Blocking app successfully checked for a new update of the " + (_isAdblockList ? "adblock" : (_isRegexList ? "regex " : "") + (_isAllowList ? "allow" : "block")) + " list: " + _listUrl.AbsoluteUri); + return false; + + default: + throw new HttpRequestException((int)httpResponse.StatusCode + " " + httpResponse.ReasonPhrase); + } + } + } + catch (Exception ex) + { + _dnsServer.WriteLog("Advanced Blocking app failed to download " + (_isAdblockList ? "adblock" : (_isRegexList ? "regex " : "") + (_isAllowList ? "allow" : "block")) + " list and will use previously downloaded file (if available): " + _listUrl.AbsoluteUri + "\r\n" + ex.ToString()); + return false; + } + } + + #endregion + + #region protected + + protected abstract void LoadListZone(); + + #endregion + + #region public + + public async Task LoadAsync() + { + if (_isLoading) + return; + + _isLoading = true; + + try + { + if (File.Exists(_listFilePath)) + { + if (!_listZoneLoaded) + { + _lastModified = File.GetLastWriteTimeUtc(_listFilePath); + LoadListZone(); + _listZoneLoaded = true; + } + } + else + { + if (await DownloadListFileAsync()) + { + LoadListZone(); + _listZoneLoaded = true; + } + } + } + finally + { + _isLoading = false; + } + } + + public async Task UpdateAsync() + { + if (await DownloadListFileAsync()) + { + LoadListZone(); + return true; + } + + return false; } #endregion #region properties - public Regex Regex - { get { return _regex; } } + public DateTime LastModified + { get { return _lastModified; } } - public IReadOnlyList BlockListUrls - { get { return _blockListUrls; } } + #endregion + } + + class BlockList : ListBase + { + #region variables + + IReadOnlyDictionary _listZone = new Dictionary(0); + + #endregion + + #region constructor + + public BlockList(IDnsServer dnsServer, Uri listUrl, bool isAllowList) + : base(dnsServer, listUrl, isAllowList, false, false) + { } + + #endregion + + #region private + + private static string PopWord(ref string line) + { + if (line.Length == 0) + return line; + + line = line.TrimStart(' ', '\t'); + + int i = line.IndexOfAny(new char[] { ' ', '\t' }); + string word; + + if (i < 0) + { + word = line; + line = ""; + } + else + { + word = line.Substring(0, i); + line = line.Substring(i + 1); + } + + return word; + } + + private Queue ReadListFile() + { + Queue domains = new Queue(); + + try + { + _dnsServer.WriteLog("Advanced Blocking app is reading " + (_isAllowList ? "allow" : "block") + " list from: " + _listUrl.AbsoluteUri); + + using (FileStream fS = new FileStream(_listFilePath, FileMode.Open, FileAccess.Read)) + { + //parse hosts file and populate block zone + StreamReader sR = new StreamReader(fS, true); + string line; + string firstWord; + string secondWord; + string hostname; + + while (true) + { + line = sR.ReadLine(); + if (line == null) + break; //eof + + line = line.TrimStart(' ', '\t'); + + if (line.Length == 0) + continue; //skip empty line + + if (line.StartsWith("#")) + continue; //skip comment line + + firstWord = PopWord(ref line); + + if (line.Length == 0) + { + hostname = firstWord; + } + else + { + secondWord = PopWord(ref line); + + if (secondWord.Length == 0) + hostname = firstWord; + else + hostname = secondWord; + } + + hostname = hostname.Trim('.').ToLower(); + + switch (hostname) + { + case "": + case "localhost": + case "localhost.localdomain": + case "local": + case "broadcasthost": + case "ip6-localhost": + case "ip6-loopback": + case "ip6-localnet": + case "ip6-mcastprefix": + case "ip6-allnodes": + case "ip6-allrouters": + case "ip6-allhosts": + continue; //skip these hostnames + } + + if (!DnsClient.IsDomainNameValid(hostname)) + continue; + + if (IPAddress.TryParse(hostname, out _)) + continue; //skip line when hostname is IP address + + domains.Enqueue(hostname); + } + } + + _dnsServer.WriteLog("Advanced Blocking app read " + (_isAllowList ? "allow" : "block") + " list file (" + domains.Count + " domains) from: " + _listUrl.AbsoluteUri); + } + catch (Exception ex) + { + _dnsServer.WriteLog("Advanced Blocking app failed to read " + (_isAllowList ? "allow" : "block") + " list from: " + _listUrl.AbsoluteUri + "\r\n" + ex.ToString()); + } + + return domains; + } + + #endregion + + #region protected + + protected override void LoadListZone() + { + Queue listQueue = ReadListFile(); + Dictionary listZone = new Dictionary(listQueue.Count); + + while (listQueue.Count > 0) + listZone.TryAdd(listQueue.Dequeue(), null); + + _listZone = listZone; + } + + #endregion + + #region public + + public bool IsZoneFound(string domain, out string foundZone) + { + return App.IsZoneFound(_listZone, domain, out foundZone); + } + + #endregion + } + + class RegexList : ListBase + { + #region variables + + IReadOnlyList _regexListZone = new List(); + + #endregion + + #region constructor + + public RegexList(IDnsServer dnsServer, Uri listUrl, bool isAllowList) + : base(dnsServer, listUrl, isAllowList, true, false) + { } + + #endregion + + #region private + + private Queue ReadRegexListFile() + { + Queue regices = new Queue(); + + try + { + _dnsServer.WriteLog("Advanced Blocking app is reading regex " + (_isAllowList ? "allow" : "block") + " list from: " + _listUrl.AbsoluteUri); + + using (FileStream fS = new FileStream(_listFilePath, FileMode.Open, FileAccess.Read)) + { + //parse hosts file and populate block zone + StreamReader sR = new StreamReader(fS, true); + string line; + + while (true) + { + line = sR.ReadLine(); + if (line == null) + break; //eof + + line = line.TrimStart(' ', '\t'); + + if (line.Length == 0) + continue; //skip empty line + + if (line.StartsWith("#")) + continue; //skip comment line + + regices.Enqueue(line); + } + } + + _dnsServer.WriteLog("Advanced Blocking app read regex " + (_isAllowList ? "allow" : "block") + " list file (" + regices.Count + " regex patterns) from: " + _listUrl.AbsoluteUri); + } + catch (Exception ex) + { + _dnsServer.WriteLog("Advanced Blocking app failed to read regex " + (_isAllowList ? "allow" : "block") + " list from: " + _listUrl.AbsoluteUri + "\r\n" + ex.ToString()); + } + + return regices; + } + + #endregion + + #region protected + + protected override void LoadListZone() + { + Queue regexPatterns = ReadRegexListFile(); + List regexListZone = new List(regexPatterns.Count); + + while (regexPatterns.Count > 0) + { + try + { + regexListZone.Add(new Regex(regexPatterns.Dequeue(), RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled)); + } + catch (RegexParseException ex) + { + _dnsServer.WriteLog(ex); + } + } + + _regexListZone = regexListZone; + } + + #endregion + + #region public + + public bool IsMatchFound(string domain, out string matchingPattern) + { + return App.IsMatchFound(_regexListZone, domain, out matchingPattern); + } + + #endregion + } + + class AdBlockList : ListBase + { + #region variables + + IReadOnlyDictionary _allowedListZone = new Dictionary(0); + IReadOnlyDictionary _blockedListZone = new Dictionary(0); + + #endregion + + #region constructor + + public AdBlockList(IDnsServer dnsServer, Uri listUrl) + : base(dnsServer, listUrl, false, false, true) + { } + + #endregion + + #region private + + private void ReadAdblockListFile(out Queue allowedDomains, out Queue blockedDomains) + { + allowedDomains = new Queue(); + blockedDomains = new Queue(); + + try + { + _dnsServer.WriteLog("Advanced Blocking app is reading adblock list from: " + _listUrl.AbsoluteUri); + + using (FileStream fS = new FileStream(_listFilePath, FileMode.Open, FileAccess.Read)) + { + //parse hosts file and populate block zone + StreamReader sR = new StreamReader(fS, true); + string line; + + while (true) + { + line = sR.ReadLine(); + if (line == null) + break; //eof + + line = line.TrimStart(' ', '\t'); + + if (line.Length == 0) + continue; //skip empty line + + if (line.StartsWith("!")) + continue; //skip comment line + + if (line.StartsWith("||")) + { + int i = line.IndexOf('^'); + if (i > -1) + { + string domain = line.Substring(2, i - 2); + string options = line.Substring(i + 1); + + if (((options.Length == 0) || (options.StartsWith("$") && (options.Contains("doc") || options.Contains("all")))) && DnsClient.IsDomainNameValid(domain)) + blockedDomains.Enqueue(domain); + } + else + { + string domain = line.Substring(2); + + if (DnsClient.IsDomainNameValid(domain)) + blockedDomains.Enqueue(domain); + } + } + else if (line.StartsWith("@@||")) + { + int i = line.IndexOf('^'); + if (i > -1) + { + string domain = line.Substring(4, i - 4); + string options = line.Substring(i + 1); + + if (((options.Length == 0) || (options.StartsWith("$") && (options.Contains("doc") || options.Contains("all")))) && DnsClient.IsDomainNameValid(domain)) + blockedDomains.Enqueue(domain); + } + else + { + string domain = line.Substring(4); + + if (DnsClient.IsDomainNameValid(domain)) + allowedDomains.Enqueue(domain); + } + } + } + } + + _dnsServer.WriteLog("Advanced Blocking app read adblock list file (" + (allowedDomains.Count + blockedDomains.Count) + " domains) from: " + _listUrl.AbsoluteUri); + } + catch (Exception ex) + { + _dnsServer.WriteLog("Advanced Blocking app failed to read adblock list from: " + _listUrl.AbsoluteUri + "\r\n" + ex.ToString()); + } + } + + #endregion + + #region protected + + protected override void LoadListZone() + { + ReadAdblockListFile(out Queue allowedDomains, out Queue blockedDomains); + + Dictionary allowedListZone = new Dictionary(allowedDomains.Count); + Dictionary blockedListZone = new Dictionary(blockedDomains.Count); + + while (allowedDomains.Count > 0) + allowedListZone.TryAdd(allowedDomains.Dequeue(), null); + + while (blockedDomains.Count > 0) + blockedListZone.TryAdd(blockedDomains.Dequeue(), null); + + _allowedListZone = allowedListZone; + _blockedListZone = blockedListZone; + } + + #endregion + + #region public + + public bool IsZoneAllowed(string domain, out string foundZone) + { + return IsZoneFound(_allowedListZone, domain, out foundZone); + } + + public bool IsZoneBlocked(string domain, out string foundZone) + { + return IsZoneFound(_blockedListZone, domain, out foundZone); + } #endregion }