diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 748401b16..6c0295cf7 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=3.0.3 +PKG_VERSION:=3.1.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 835781351..d76c15600 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -4,7 +4,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other wlan gadget...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router. When the dns server on your router receives dns requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if domain name is unable to resolved using the dns server, a condition called the 'NXDOMAIN' occurred. ## Main Features -* support of the following domain block list sources (free for private usage, for commercial use please check their individual licenses): +* support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses): * [adaway](https://adaway.org) * => infrequent updates, approx. 400 entries (enabled by default) * [adguard](https://adguard.com) @@ -61,28 +61,28 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 440 entries * zero-conf like automatic installation & setup, usually no manual changes needed * simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that -* supports five different dns backends / block list formats: dnsmasq, unbound, named (bind), kresd and dnscrypt-proxy +* supports five different dns backends / blocklist formats: dnsmasq, unbound, named (bind), kresd and dnscrypt-proxy * automatically selects uclient-fetch or wget as download utility (other tools like curl or aria2c are supported as well) -* provides 'http only' mode without installed ssl library for all non-SSL block list sources +* provides 'http only' mode without installed ssl library for all non-SSL blocklist sources * supports a wide range of router modes, even AP modes are supported * full IPv4 and IPv6 support -* provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the block list and lowers the memory footprint for the dns backends -* each block list source will be updated and processed separately -* block list source parsing by fast & flexible regex rulesets -* overall duplicate removal in central block list (adb_list.overall) +* provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the blocklist and lowers the memory footprint for the dns backends +* blocklist source parsing by fast & flexible regex rulesets +* overall duplicate removal in central blocklist 'adb_list.overall' * additional whitelist for manual overrides, located by default in /etc/adblock/adblock.whitelist -* quality checks during block list update to ensure a reliable dns backend service +* quality checks during blocklist update to ensure a reliable dns backend service * minimal status & error logging to syslog, enable debug logging to receive more output * procd based init system support (start/stop/restart/reload/suspend/resume/query/status) * procd network interface trigger support or classic time based startup -* suspend & resume adblock actions temporarily without block list reloading +* suspend & resume adblock actions temporarily without blocklist reloading * output comprehensive runtime information via LuCI or via 'status' init command * query function to quickly identify blocked (sub-)domains, e.g. for whitelisting * strong LuCI support * optional: force dns requests to local resolver * optional: force overall sort / duplicate removal for low memory devices (handle with care!) -* optional: 'manual mode' to re-use blocklist backups during startup, get fresh lists only via manual reload or restart action -* optional: automatic block list backup & restore, they will be used in case of download errors or during startup in manual mode +* optional: automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode +* optional: 'backup mode' to re-use blocklist backups during startup, get fresh lists only via reload or restart action +* optional: 'whitelist mode' to block access to all domains except those explicitly listed in the whitelist file * optional: add new adblock sources on your own via uci config ## Prerequisites @@ -99,21 +99,21 @@ A lot of people already use adblocker plugins within their desktop browsers, but * control the adblock service manually with _/etc/init.d/adblock_ start/stop/restart/reload/suspend/resume/status or use the LuCI frontend ## LuCI adblock companion package -* for easy management of the various block list sources and all other adblock options you should use the provided LuCI frontend +* for easy management of the various blocklist sources and all other adblock options you should use the provided LuCI frontend * install 'luci-app-adblock' (_opkg install luci-app-adblock_) * the application is located in LuCI under 'Services' menu ## Tweaks * **runtime information:** the adblock status is available via _/etc/init.d/adblock status_ (see example below) * **debug logging:** for script debugging please set the config option 'adb\_debug' to '1' and check the runtime output with _logread -e "adblock"_ -* **storage expansion:** to process and store all block list sources at once it might helpful to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details +* **storage expansion:** to process and store all blocklist sources at once it might helpful to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details * **add white- / blacklist entries:** add domain white- or blacklist entries to always-allow or -deny certain (sub) domains, by default both lists are empty and located in _/etc/adblock_. Please add one domain per line - ip addresses, wildcards & regex are _not_ allowed (see example below) -* **backup & restore block lists:** enable this feature, to restore automatically the latest compressed backup of your block lists in case of any processing error (e.g. a single block list source is not available during update). Please use an (external) solid partition and _not_ your volatile router temp directory for this +* **backup & restore blocklists:** enable this feature, to restore automatically the latest compressed backup of your blocklists in case of any processing error (e.g. a single blocklist source is not available during update). Please use an (external) solid partition and _not_ your volatile router temp directory for this * **scheduled list updates:** for a scheduled call of the adblock service add an appropriate crontab entry (see example below) * **change startup behaviour:** by default the startup will be triggered by the 'wan' procd interface trigger. Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface. * **suspend & resume adblocking:** to quickly switch the adblock service 'on' or 'off', simply use _/etc/init.d/adblock [suspend|resume]_ -* **domain query:** to query the active block list for a specific domain, please run _/etc/init.d/adblock query ``_ (see example below) -* **add new list sources:** you could add new block list sources on your own via uci config, all you need is a source url and an awk one-liner (see example below) +* **domain query:** to query the active blocklist for a specific domain, please run _/etc/init.d/adblock query ``_ (see example below) +* **add new list sources:** you could add new blocklist sources on your own via uci config, all you need is a source url and an awk one-liner (see example below) * **disable active dns probing in windows 10:** to prevent a yellow exclamation mark on your internet connection icon (which wrongly means connected, but no internet), please change the following registry key/value from "1" to "0" _HKLM\SYSTEM\CurrentControlSet\Services\NlaSvc\Parameters\Internet\EnableActiveProbing_ ## Further adblock config options @@ -128,70 +128,44 @@ A lot of people already use adblocker plugins within their desktop browsers, but * adb\_fetchparm => options for the download utility, see example below (default: not set, use wget default options) * adb\_forcedns => force dns requests to local resolver (default: '0', disabled) * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (default: '0', disabled) - * adb\_manmode => do not automatically update block lists during startup, use backups instead (default: '0', disabled) + * adb\_backup_mode => do not automatically update blocklists during startup, use backups instead (default: '0', disabled) + * adb\_whitelist_mode => block access to all domains except those explicitly listed in the whitelist file (default: '0', disabled) ## Examples **change default dns backend to 'unbound':** -Adblock deposits the sorted and filtered block list (adb_list.overall) in '/var/lib/unbound' where unbound can find them in its jail. -If you use manual configuration for unbound, then just include the following line in your 'server' clause: +Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' where unbound can find them in its jail. If you use manual configuration for unbound, then just include the following line in your 'server' clause: 

   include: "/var/lib/unbound/adb_list.overall"
**change default dns backend to 'named' (bind):** -Adblock deposits the sorted and filtered block list (adb_list.overall) in '/var/lib/bind' where bind can find them. -To use the block list please modify the following bind configuration files: +Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind'. To use the blocklist please modify '/etc/bind/named.conf': 

-change '/etc/bind/named.conf', in the 'options' namespace add:
+in the 'options' namespace add:
   response-policy { zone "rpz"; };
 
 and at the end of the file add:
   zone "rpz" {
     type master;
-    file "/etc/bind/db.rpz";
+    file "/var/lib/bind/adb_list.overall";
     allow-query { none; };
     allow-transfer { none; };
   };
-
-create the new file '/etc/bind/db.rpz' and add:
-  $TTL 2h
-  $ORIGIN rpz.
-  @ SOA localhost. root.localhost. (1 6h 1h 1w 2h)
-  NS localhost.
-
-  $INCLUDE /var/lib/bind/adb_list.overall
**change default dns backend to 'kresd':** -The knot-resolver (kresd) is only available on turris omnia devices. Currently there's no package for kresd in the official LEDE / OpenWrt package repository. -Adblock deposits the sorted and filtered block list (adb_list.overall) in '/etc/kresd' where kresd can find them. -To use the block list please create/modify the following kresd configuration files: +The knot-resolver (kresd) is only available on turris omnia devices. Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd'. To use the blocklist please modify '/etc/config/resolver': 

-TurrisOS > 3.6:
-  edit '/etc/config/resolver' and change / uncomment the following options:
-    forward_upstream '0'
     list rpz_file '/etc/kresd/adb_list.overall'
-
-TurrisOS < 3.6:
-  edit '/etc/config/resolver' and change / uncomment the following options:
-   forward_upstream '0'
-   option include_config '/etc/kresd/custom.conf'
-
-  create '/etc/kresd/custom.conf' and add:
-    policy.add(policy.rpz(policy.DENY, '/etc/kresd/adb_list.overall'))
-    policy.add(policy.all(policy.FORWARD({'8.8.8.8', '8.8.4.4'})))
**change default dns backend to 'dnscrypt-proxy':** -Adblock deposits the sorted and filtered block list (adb_list.overall) by default in '/tmp' where DNSCrypt-Proxy can find them. -The blacklist option is not supported by default, because DNSCrypt-Proxy is compiled without plugins support. -Take a custom LEDE build with plugins support to use this feature: +The required 'blacklist' option of dnscrypt-proxy is not enabled by default, because the package will be compiled without plugins support. Take a custom LEDE build with plugins support to use this feature. Adblock deposits the final blocklist 'adb_list.overall' in '/tmp'. To use the blocklist please modify '/etc/config/dnscrypt-proxy' per instance: 

-edit '/etc/config/dnscrypt-proxy' and add the following option per dnscrypt-proxy instance:
   list blacklist 'domains:/tmp/adb_list.overall'
@@ -219,12 +193,12 @@ curl: /etc/init.d/adblock status ::: adblock runtime information + adblock_status : enabled - + adblock_version : 3.0.0 - + blocked_domains : 37406 + + adblock_version : 3.1.0 + + overall_domains : 5117 + fetch_utility : wget (built-in) + dns_backend : kresd (/etc/kresd) - + last_rundate : 08.09.2017 21:21:21 - + system_release : OpenWrt omnia 15.05 + + last_rundate : 03.11.2017 22:57:41 + + system_release : Turris Omnia, OpenWrt omnia 15.05/3.8.4 **cronjob for a regular block list update (/etc/crontabs/root):** @@ -250,7 +224,7 @@ This entry does not block: 

 here.com
 
-This entry removes the following (sub)domains from the block list:
+This entry removes the following (sub)domains from the blocklist:
   maps.here.com
   here.com
 
@@ -259,51 +233,40 @@ This entry does not remove:
   www.adwhere.com
-**query active block list for a certain (sub-)domain, e.g. for whitelisting:** +**query the active blocklist for a certain (sub-)domain, e.g. for whitelisting:** +The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain. For every (sub-)domain it returns the first ten relevant results. 

-/etc/init.d/adblock query example.www.doubleclick.net
-root@turris:~# /etc/init.d/adblock query example.www.doubleclick.net
-::: max. ten results for domain 'example.www.doubleclick.net'
+/etc/init.d/adblock query www.example.google.com
+::: max. ten results for domain 'www.example.google.com'
   - no match
-::: max. ten results for domain 'www.doubleclick.net'
+::: max. ten results for domain 'example.google.com'
   - no match
-::: max. ten results for domain 'doubleclick.net'
-  + doubleclick.net
-
-The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain.
-For every (sub-)domain it returns the first ten relevant results.
-In the example above whitelist "doubleclick.net" to free the submitted domain.
+::: max. ten results for domain 'google.com'
+  + analytics.google.com
+  + googleadapis.l.google.com
+  + pagead.l.google.com
+  + partnerad.l.google.com
+  + ssl-google-analytics.l.google.com
+  + video-stats.video.google.com
+  + www-google-analytics.l.google.com
-**add a new block list source:** -

-1. the easy way ...
-example: https://easylist-downloads.adblockplus.org/rolist+easylist.txt
-adblock already supports an easylist source, called 'ruadlist'. To add the additional local easylist
-as a new source, copy the existing config source 'ruadlist' section and change only
+**add a new blocklist source:**
+1. the easy way ...  
+example: https://easylist-downloads.adblockplus.org/rolist+easylist.txt  
+Adblock already supports an easylist source, called 'reg_ru'. To add the additional local easylist as a new source, copy the existing config source section and change only
 the source name, the url and the description - that's all!
-
-config source 'rolist'
+

+config source 'reg_ro'
   option enabled '0'
   option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt'
-  option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}'
-  option adb_src_desc 'focus on romanian ad related domains plus generic easylist additions, weekly updates, approx. 600 entries'
+  option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}'
+  option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 9.400 entries'
+

 
-2. a bit harder ...
-to add a really new source with different domain/host format you have to write a suitable
-awk one-liner on your own, so basic awk skills are needed. As a starting point check the already
-existing awk strings (adb_src_rset) in adblock config, maybe you need only small changes for your individual list.
-Download the desired list and test your new awk string locally with:
-  cat new.list | awk 'fs__individual search__search core__result'
-  'fs' => field separator (optional)
-  'individual search' => individual search part to filter out needless list information
-  'search core' => always '([A-Za-z0-9_-]+\.){1,}[A-Za-z]+', this is part of all list sources and should be unchanged
-  'result' => always '{print tolower(\$n)}', only the output column 'n' may vary
-the output result should be a sequential list with one domain/host per line - nothing more.
+2. a bit harder ...  
+To add a really new source with different domain/host format you have to write a suitable awk one-liner on your own, so basic awk skills are needed. As a starting point check the already existing awk rulesets 'adb_src_rset' in the config file, probably you need only small changes for your individual list. Download the desired list and test your new awk string locally. The output result should be a sequential list with one domain/host per line - nothing more. If your awk one-liner works quite well, add a new source section to the adblock config file and test the new source.  
 
-If your awk one-liner works quite well, add a new source section in adblock config and test your new source
-
- ## Support Please join the adblock discussion in this [forum thread](https://forum.lede-project.org/t/adblock-2-x-support-thread/507) or contact me by mail diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index eb9c6bd08..e7e8092a8 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -3,136 +3,134 @@ config adblock 'global' option adb_enabled '0' + option adb_dns 'dnsmasq' + option adb_trigger 'wan' + +config adblock 'extra' option adb_debug '0' option adb_forcesrt '0' option adb_forcedns '0' - option adb_dns 'dnsmasq' - option adb_trigger 'wan' - option adb_whitelist '/etc/adblock/adblock.whitelist' - option adb_whitelist_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}' option adb_backup '0' - option adb_backupdir '/mnt' - option adb_rtfile '/tmp/adb_runtime.json' config source 'adaway' option enabled '1' option adb_src 'https://adaway.org/hosts.txt' - option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' + option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' option adb_src_desc 'focus on mobile ads, infrequent updates, approx. 400 entries' config source 'adguard' option enabled '0' - option adb_src 'https://raw.githubusercontent.com/AdguardTeam/AdguardDNS/master/Filters/filter.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'combined adguard dns filter list, approx. 12.000 entries' + option adb_src 'https://filters.adtidy.org/windows/filters/15.txt' + option adb_src_rset 'BEGIN{FS=\"[/|^|\r]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+([\/\^\r]|$)/{print tolower(\$3)}' + option adb_src_desc 'combined adguard dns filter list, frequent updates, approx. 15.700 entries' config source 'bitcoin' option enabled '0' option adb_src 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt' - option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'focus on malicious bitcoin mining sites, infrequent updates, approx. 15 entries' + option adb_src_rset '\$0~/^0\.0\.0\.0[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'focus on malicious bitcoin mining sites, infrequent updates, approx. 20 entries' config source 'blacklist' option enabled '0' option adb_src '/etc/adblock/adblock.blacklist' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'static local domain blacklist (always deny these domains)' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'static local domain blacklist, always deny these domains' config source 'disconnect' option enabled '1' option adb_src 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'mozilla driven blocklist, numerous updates on the same day, approx. 6.500 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'mozilla driven blocklist, numerous updates on the same day, approx. 4.600 entries' config source 'dshield' option enabled '0' option adb_src 'https://www.dshield.org/feeds/suspiciousdomains_Low.txt' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'generic blocklist, daily updates, approx. 4.500 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'generic blocklist, daily updates, approx. 3.500 entries' config source 'feodo' option enabled '0' option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' option adb_src_desc 'focus on feodo botnet, daily updates, approx. 0-10 entries' config source 'hphosts' option enabled '0' option adb_src 'https://hosts-file.net/ad_servers.txt' - option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'broad blocklist, monthly updates, approx. 50.000 entries' + option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist, monthly updates, approx. 19.200 entries' config source 'malware' option enabled '0' option adb_src 'https://mirror.cedia.org.ec/malwaredomains/justdomains' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'broad blocklist, daily updates, approx. 16.000 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'broad blocklist, daily updates, approx. 18.300 entries' config source 'malwarelist' option enabled '0' option adb_src 'http://www.malwaredomainlist.com/hostslist/hosts.txt' - option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'focus on malware, daily updates, approx. 1.500 entries' + option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'focus on malware, daily updates, approx. 1.200 entries' config source 'openphish' option enabled '0' option adb_src 'https://openphish.com/feed.txt' - option adb_src_rset '{FS=\"/\"} \$3 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$3)}' - option adb_src_desc 'focus on phishing, numerous updates on the same day, approx. 1.800 entries' + option adb_src_rset 'BEGIN{FS=\"/\"}\$0~/^http[s]?:\/\/([[:alnum:]_-]+\.){1,}[[:alpha:]]+(\/|$)/{print tolower(\$3)}' + option adb_src_desc 'focus on phishing, numerous updates on the same day, approx. 2.400 entries' config source 'ransomware' option enabled '0' option adb_src 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 130 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|\r|$)/{print tolower(\$1)}' + option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 1900 entries' config source 'reg_cn' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'focus on chinese ads, daily updates, approx. 1.600 entries' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_desc 'focus on chinese ads plus generic easylist additions, daily updates, approx. 11.700 entries' config source 'reg_id' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/abpindo+easylist.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'focus on indonesian ads plus generic easylist additions, weekly updates, approx. 800 entries' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_desc 'focus on indonesian ads plus generic easylist additions, weekly updates, approx. 9.600 entries' config source 'reg_nl' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/easylistdutch+easylist.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'focus on dutch ads plus generic easylist additions, weekly updates, approx. 1300 entries' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_desc 'focus on dutch ads plus generic easylist additions, weekly updates, approx. 9.400 entries' config source 'reg_pl' option enabled '0' option adb_src 'http://adblocklist.org/adblock-pxf-polish.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'focus on polish ads, daily updates, approx. 50 entries' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_desc 'focus on polish ads, daily updates, approx. 90 entries' config source 'reg_ro' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 600 entries' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 9.400 entries' config source 'reg_ru' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt' - option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' - option adb_src_desc 'focus on russian ads plus generic easylist additions, weekly updates, approx. 2.000 entries' + option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' + option adb_src_desc 'focus on russian ads plus generic easylist additions, weekly updates, approx. 14.500 entries' config source 'securemecca' option enabled '0' option adb_src 'http://securemecca.com/Downloads/hosts.txt' - option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'broad blocklist, infrequent updates, approx. 25.000 entries' + option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist, infrequent updates, approx. 13.700 entries' config source 'shalla' option enabled '0' option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz' - option adb_src_rset '{FS=\"/\"} \$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'broad blocklist subdivided in different categories, daily updates, approx. 32.000 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'broad blocklist subdivided in different categories, daily updates, approx. 31.700 entries' list adb_src_cat 'adv' list adb_src_cat 'costtraps' list adb_src_cat 'spyware' @@ -142,41 +140,41 @@ config source 'shalla' config source 'spam404' option enabled '0' option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'generic blocklist, infrequent updates, approx. 5.000 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'generic blocklist, infrequent updates, approx. 6.000 entries' config source 'sysctl' option enabled '0' option adb_src 'http://sysctl.org/cameleon/hosts' - option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'broad blocklist, weekly updates, approx. 21.000 entries' + option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist, weekly updates, approx. 16.500 entries' config source 'whocares' option enabled '0' option adb_src 'http://someonewhocares.org/hosts/hosts' - option adb_src_rset '\$0 ~/^127\.0\.0\.1[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'broad blocklist, weekly updates, approx. 12.000 entries' + option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist, weekly updates, approx. 10.000 entries' config source 'winspy' option enabled '0' option adb_src 'https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/win10/spy.txt' - option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'focus on windows spy & telemetry, infrequent updates, approx. 140 entries' + option adb_src_rset '\$0~/^0\.0\.0\.0[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'focus on windows spy & telemetry domains, infrequent updates, approx. 300 entries' config source 'winhelp' option enabled '0' option adb_src 'http://winhelp2002.mvps.org/hosts.txt' - option adb_src_rset '\$0 ~/^0\.0\.0\.0[ \t]+([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$2)}' - option adb_src_desc 'broad blocklist, infrequent updates, approx. 15.000 entries' + option adb_src_rset '\$0~/^0\.0\.0\.0[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' + option adb_src_desc 'broad blocklist, infrequent updates, approx. 13.000 entries' config source 'yoyo' option enabled '1' option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'focus on ad related domains, weekly updates, approx. 2.500 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'focus on ad related domains, weekly updates, approx. 2.400 entries' config source 'zeus' option enabled '0' option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'focus on zeus botnet, daily updates, approx. 440 entries' + option adb_src_rset '\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}' + option adb_src_desc 'focus on zeus botnet, daily updates, approx. 400 entries' diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index cd0357617..e628637a7 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,11 +10,12 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="3.0.3" +adb_ver="3.1.0" adb_sysver="unknown" adb_enabled=0 adb_debug=0 -adb_manmode=0 +adb_backup_mode=0 +adb_whitelist_mode=0 adb_forcesrt=0 adb_forcedns=0 adb_triggerdelay=0 @@ -25,6 +26,7 @@ adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --n adb_dns="dnsmasq" adb_dnsprefix="adb_list" adb_dnsfile="${adb_dnsprefix}.overall" +adb_whitelist="/etc/adblock/adblock.whitelist" adb_rtfile="/tmp/adb_runtime.json" adb_action="${1:-"start"}" adb_cnt=0 @@ -61,7 +63,7 @@ f_envload() f_log "error" "system libraries not found" fi - # parse global section by callback + # parse 'global' and 'extra' section by callback # config_cb() { @@ -79,7 +81,7 @@ f_envload() fi } - # parse 'source' sections + # parse 'source' typed sections # parse_config() { @@ -104,33 +106,66 @@ f_envload() # case "${adb_dns}" in dnsmasq) - adb_dnsuser="${adb_dns}" + adb_dnsuser="${adb_dnsuser:-"dnsmasq"}" adb_dnsdir="${adb_dnsdir:-"/tmp/dnsmasq.d"}" adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'" + if [ ${adb_whitelist_mode} -eq 1 ] + then + adb_dnsformat="awk '{print \"local=/\"\$0\"/#\"}'" + adb_dnsblock="local=/#/" + fi ;; unbound) - adb_dnsuser="${adb_dns}" + adb_dnsuser="${adb_dnsuser:-"unbound"}" adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" + if [ ${adb_whitelist_mode} -eq 1 ] + then + adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 transparent\"}'" + adb_dnsblock="local-zone: \".\" static" + fi ;; named) - adb_dnsuser="bind" + adb_dnsuser="${adb_dnsuser:-"bind"}" adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" - adb_dnsformat="awk '{print \"\"\$0\" IN CNAME .\n*.\"\$0\" IN CNAME .\"}'" + adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)"$'\n'" IN NS localhost." + adb_dnsformat="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" + if [ ${adb_whitelist_mode} -eq 1 ] + then + adb_dnsformat="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" + adb_dnsblock="* CNAME ." + fi ;; kresd) - adb_dnsuser="root" + adb_dnsuser="${adb_dnsuser:-"root"}" adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" + adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)"$'\n'" IN NS localhost." adb_dnsformat="awk '{print \"\"\$0\" CNAME .\n*.\"\$0\" CNAME .\"}'" - adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (2 6h 1h 1w 2h)"$'\n'" IN NS localhost." + if [ ${adb_whitelist_mode} -eq 1 ] + then + adb_dnsformat="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" + adb_dnsblock="* CNAME ." + fi ;; dnscrypt-proxy) - adb_dnsuser="nobody" + adb_dnsuser="${adb_dnsuser:-"nobody"}" adb_dnsdir="${adb_dnsdir:-"/tmp"}" adb_dnsformat="awk '{print \$0}'" ;; esac + if [ ${adb_enabled} -ne 1 ] + then + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] + then + f_rmdns + f_dnsrestart + fi + f_jsnupdate + f_log "info " "adblock is currently disabled, please set adb_enabled to '1' to use this service" + exit 0 + fi + if [ -d "${adb_dnsdir}" ] && [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] then > "${adb_dnsdir}/${adb_dnsfile}" @@ -193,20 +228,6 @@ f_envcheck() { local ssl_lib - # check 'enabled' option - # - if [ ${adb_enabled} -ne 1 ] - then - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] - then - f_rmdns - f_dnsrestart - fi - f_jsnupdate - f_log "info " "adblock is currently disabled, please set adb_enabled to '1' to use this service" - exit 0 - fi - # check fetch utility # ssl_lib="-" @@ -241,18 +262,12 @@ f_envcheck() fi adb_fetchinfo="${adb_fetch##*/} (${ssl_lib})" - # create adblock temp file/directory + # initialize temp files and directories # adb_tmpload="$(mktemp -tu)" adb_tmpfile="$(mktemp -tu)" adb_tmpdir="$(mktemp -p /tmp -d)" - - # prepare whitelist entries - # - if [ -s "${adb_whitelist}" ] && [ -n "${adb_whitelist_rset}" ] - then - awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" - fi + > "${adb_tmpdir}/tmp.whitelist" } # f_rmtemp: remove temporary files & directories @@ -300,7 +315,7 @@ f_dnsrestart() return 1 } -# f_list: backup/restore/remove block lists +# f_list: backup/restore/remove blocklists # f_list() { @@ -388,13 +403,14 @@ f_switch() then cat "${source}" > "${target}" > "${source}" + chown "${adb_dnsuser}" "${target}" 2>/dev/null f_dnsrestart f_jsnupdate f_log "info " "adblock processing ${status}" fi } -# f_query: query block list for certain (sub-)domains +# f_query: query blocklist for certain (sub-)domains # f_query() { @@ -404,7 +420,7 @@ f_query() if [ ! -s "${adb_dnsdir}/${adb_dnsfile}" ] then - printf "%s\n" "::: no active block list found, please start / resume adblock first" + printf "%s\n" "::: no active blocklist found, please start / resume adblock first" elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then printf "%s\n" "::: invalid domain input, please submit a single domain, e.g. 'doubleclick.net'" @@ -417,7 +433,7 @@ f_query() then result="$(awk -F '/|\"' "/[\/\"\.]${search}/{i++;{printf(\" + %s\n\",\$2)};if(i>9){exit}}" "${adb_dnsfile}")" else - result="$(awk "/(^[^\*][a-z]*[\.]+${search}|^${search})/{i++;{printf(\" + %s\n\",\$1)};if(i>9){exit}}" "${adb_dnsfile}")" + result="$(awk "/(^[^\*][[:alpha:]]*[\.]+${search}|^${search})/{i++;{printf(\" + %s\n\",\$1)};if(i>9){exit}}" "${adb_dnsfile}")" fi printf "%s\n" "::: max. ten results for domain '${domain}'" printf "%s\n" "${result:-" - no match"}" @@ -450,7 +466,7 @@ f_jsnupdate() then adb_cnt="$(( ( $(wc -l < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^") ) / 2 ))" else - adb_cnt="$(( $(wc -l < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^") ))" + adb_cnt="$(wc -l < "${adb_dnsdir}/${adb_dnsfile}")" fi fi fi @@ -466,7 +482,7 @@ f_jsnupdate() json_add_object "data" json_add_string "adblock_status" "${status}" json_add_string "adblock_version" "${adb_ver}" - json_add_string "blocked_domains" "${adb_cnt}" + json_add_string "overall_domains" "${adb_cnt}" json_add_string "fetch_utility" "${adb_fetchinfo}" json_add_string "dns_backend" "${adb_dns} (${adb_dnsdir})" json_add_string "last_rundate" "${rundate}" @@ -520,7 +536,7 @@ f_log() fi } -# main function for block list processing +# main function for blocklist processing # f_main() { @@ -528,8 +544,48 @@ f_main() local mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" f_log "info " "start adblock processing ..." - f_log "debug" "action: ${adb_action}, manual_mode:${adb_manmode}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, mem_total: ${mem_total}, force_srt/_dns: ${adb_forcesrt}/${adb_forcedns}" + f_log "debug" "action: ${adb_action}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, whitelist_mode: ${adb_whitelist_mode}, force_srt/_dns: ${adb_forcesrt}/${adb_forcedns}, mem_total: ${mem_total}" > "${adb_rtfile}" + > "${adb_dnsdir}/.${adb_dnsfile}" + + # prepare whitelist entries + # + if [ -s "${adb_whitelist}" ] + then + if [ ${adb_whitelist_mode} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] + then + adb_whitelist_rset="\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" + else + adb_whitelist_rset="\$0~/^([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\.\",\"\\\.\",\$1);print tolower(\"^\"\$1\"\\\|\\\.\"\$1)}" + fi + awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" + fi + + # whitelist mode + # + if [ ${adb_whitelist_mode} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] + then + f_tldcompression "${adb_tmpdir}/tmp.whitelist" + eval "${adb_dnsformat}" "${adb_tmpdir}/tmp.whitelist" > "${adb_dnsdir}/${adb_dnsfile}" + printf '%s\n' "${adb_dnsblock}" >> "${adb_dnsdir}/${adb_dnsfile}" + if [ -n "${adb_dnsheader}" ] + then + printf '%s\n' "${adb_dnsheader}" | cat - "${adb_dnsdir}/${adb_dnsfile}" > "${adb_tmpdir}/${adb_dnsfile}" + cat "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" + fi + f_dnsrestart + if [ ${?} -eq 0 ] + then + f_jsnupdate "${adb_cnt}" + f_log "info " "whitelist with overall ${adb_cnt} domains loaded successfully (${adb_sysver})" + else + f_log "error" "dns backend restart with active whitelist failed" + fi + return + fi + + # normal & backup mode + # for src_name in ${adb_sources} do eval "enabled=\"\${enabled_${src_name}}\"" @@ -548,9 +604,9 @@ f_main() continue fi - # manual / backup mode + # backup mode # - if [ ${adb_manmode} -eq 1 ] && [ "${adb_action}" = "start" ] && [ "${src_name}" != "blacklist" ] + if [ ${adb_backup_mode} -eq 1 ] && [ "${adb_action}" = "start" ] && [ "${src_name}" != "blacklist" ] then f_list restore if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] @@ -560,7 +616,7 @@ f_main() fi fi - # download block list + # download blocklist # if [ "${src_name}" = "blacklist" ] && [ -s "${url}" ] then @@ -652,9 +708,9 @@ f_main() if [ ${?} -eq 0 ] then f_jsnupdate "${adb_cnt}" - f_log "info " "block list with overall ${adb_cnt} domains loaded successfully (${adb_sysver})" + f_log "info " "blocklist with overall ${adb_cnt} domains loaded successfully (${adb_sysver})" else - f_log "error" "dns backend restart with active block list failed" + f_log "error" "dns backend restart with active blocklist failed" fi }