From 215ffbe35753484c3cd0807341cd82bdba56884f Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Fri, 26 Feb 2021 20:52:55 +0100 Subject: [PATCH] adblock: update 4.1.0 * major source changes: * split oisd.nl in basic and full variant * add swedish regional list * made archive categories for shallalist and utcapitole selectable via LuCI * made all list variants of energized and stevenblack selectable via LuCI * removed dns filereset mode Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 8 +- net/adblock/files/README.md | 75 +++-- net/adblock/files/adblock.categories | 145 ++++++++++ net/adblock/files/adblock.init | 253 ++++++++++++----- net/adblock/files/adblock.monitor | 40 --- net/adblock/files/adblock.sh | 404 ++++++++++++++------------- net/adblock/files/adblock.sources | 112 ++------ 7 files changed, 599 insertions(+), 438 deletions(-) create mode 100644 net/adblock/files/adblock.categories delete mode 100755 net/adblock/files/adblock.monitor diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 81be42b7e..94a4df594 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -1,13 +1,13 @@ # -# Copyright (c) 2015-2020 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2021 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=4.0.8 -PKG_RELEASE:=3 +PKG_VERSION:=4.1.0 +PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken @@ -55,9 +55,9 @@ define Package/adblock/install $(INSTALL_DIR) $(1)/etc/adblock $(INSTALL_BIN) ./files/adblock.mail $(1)/etc/adblock - $(INSTALL_BIN) ./files/adblock.monitor $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.categories $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.sources $(1)/etc/adblock gzip -9 $(1)/etc/adblock/adblock.sources endef diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index bf6861349..2e4e4cafa 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -18,17 +18,13 @@ A lot of people already use adblocker plugins within their desktop browsers, but | anudeep | | M | compilation | [Link](https://github.com/anudeepND/blacklist) | | bitcoin | | S | mining | [Link](https://github.com/hoshsadiq/adblock-nocoin-list) | | disconnect | x | S | general | [Link](https://disconnect.me) | -| energized_blugo | | XL | compilation | [Link](https://energized.pro) | -| energized_blu | | XL | compilation | [Link](https://energized.pro) | -| energized_porn | | XXL | compilation+porn | [Link](https://energized.pro) | -| energized_ultimate | | XXL | compilation | [Link](https://energized.pro) | -| energized_unified | | XXL | compilation | [Link](https://energized.pro) | +| energized | | VAR | compilation | [Link](https://energized.pro) | | firetv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | games_tracking | | S | tracking | [Link](https://www.gameindustry.eu) | | gaming | | S | gaming | [Link](https://github.com/abyssin/pihole-blocklist) | -| malwaredomains | | M | malware | [Link](https://malwaredomains.com) | | notracking | | XL | tracking | [Link](https://github.com/notracking/hosts-blocklists) | -| oisd_nl | | XXL | general | [Link](https://oisd.nl) | +| oisd_basic | | L | general | [Link](https://oisd.nl) | +| oisd_full | | XXL | general | [Link](https://oisd.nl) | | openphish | | S | phishing | [Link](https://openphish.com) | | phishing_army | | S | phishing | [Link](https://phishing.army) | | reg_cn | | M | reg_china | [Link](https://easylist.to) | @@ -45,16 +41,14 @@ A lot of people already use adblocker plugins within their desktop browsers, but | reg_pl2 | | S | reg_poland | [Link](https://www.certyficate.it) | | reg_ro | | M | reg_romania | [Link](https://easylist.to) | | reg_ru | | M | reg_russia | [Link](https://easylist.to) | +| reg_se | | M | reg_sweden | [Link](https://github.com/lassekongo83/Frellwits-filter-lists) | | reg_vn | | S | reg_vietnam | [Link](https://bigdargon.github.io/hostsVN) | -| shallalist | | L | general | [Link](https://www.shallalist.de) | -| shallalist_porn | | XXL | general+porn | [Link](https://www.shallalist.de) | +| shallalist | | VAR | general | [Link](https://www.shallalist.de) | | smarttv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | spam404 | | S | general | [Link](https://github.com/Dawsey21) | -| stevenblack | | L | compilation | [Link](https://github.com/StevenBlack/hosts) | -| stevenblack_porn | | L | compilation+porn | [Link](https://github.com/StevenBlack/hosts) | +| stevenblack | | VAR | compilation | [Link](https://github.com/StevenBlack/hosts) | | stopforumspam | | S | spam | [Link](https://www.stopforumspam.com) | -| utcapitole | | L | general | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | -| utcapitole_porn | | XXL | general+porn | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | +| utcapitole | | VAR | general | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | | wally3k | | S | compilation | [Link](https://firebog.net/about) | | whocares | | M | general | [Link](https://someonewhocares.org) | | winhelp | | S | general | [Link](https://winhelp2002.mvps.org) | @@ -117,26 +111,28 @@ A lot of people already use adblocker plugins within their desktop browsers, but ## Adblock CLI Options * All important adblock functions are accessible via CLI as well. -Please note: The 'status' command in 19.07.x and TurrisOS is only available via 'status_service' 

-/etc/init.d/adblock 
+~# /etc/init.d/adblock
 Syntax: /etc/init.d/adblock [command]
 
 Available commands:
-	start	Start the service
-	stop	Stop the service
-	restart	Restart the service
-	reload	Reload configuration files (or restart if service does not implement reload)
-	enable	Enable service autostart
-	disable	Disable service autostart
-	running	Check if service is running
-	status	Service status
-	suspend	Suspend adblock processing
-	resume	Resume adblock processing
-	query	<domain> Query active blocklists and backups for a specific domain
-	report	[<search>] Print DNS statistics with an optional search parameter
-	list	[[<add>|<remove>] [source(s)]] List available adblock sources or add/remove them from config
-	timer	<action> <hour> [<minute>] [<weekday>] Set a cron based update interval
+	start           Start the service
+	stop            Stop the service
+	restart         Restart the service
+	reload          Reload configuration files (or restart if service does not implement reload)
+	enable          Enable service autostart
+	disable         Disable service autostart
+	enabled         Check if service is started on boot
+	suspend         Suspend adblock processing
+	resume          Resume adblock processing
+	query           <domain> Query active blocklists and backups for a specific domain
+	report          [<search>] Print DNS statistics with an optional search parameter
+	list            [<add>|<add_sha>|<add_utc>|<add_eng>|<add_stb>|<remove>|<remove_sha>|<remove_utc>|<remove_eng>|<remove_stb>] <source(s)> List/Edit available sources
+	timer           [<add> <tasks> <hour> [<minute>] [<weekday>]]|[<remove> <line no.>] List/Edit cron update intervals
+	version         Print version information
+	running         Check if service is running
+	status          Service status
+	trace           Start with syscall trace
## Adblock Config Options @@ -159,7 +155,6 @@ Available commands: | adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | | adb_dnstimeout | 10 | timeout in seconds to wait for a successful DNS backend restart | | adb_dnsinstance | 0, first instance | set to the relevant dns backend instance used by adblock (dnsmasq only) | -| adb_dnsfilereset | 0, disabled | set to 1 to purge the final DNS blocklist file after DNS backend loading | | adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | | adb_dnsinotify | -, not set | set to 1 to prevent adblock triggered restarts for DNS backends with autoload functions | | adb_dnsallow | -, not set | set to 1 to disable selective DNS whitelisting (RPZ pass through) | @@ -236,19 +231,19 @@ Finally enable E-Mail support and add a valid E-Mail receiver address in LuCI. In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_ (in 19.07 and TurrisOS): 

-/etc/init.d/adblock status_service
+~# /etc/init.d/adblock status
 ::: adblock runtime information
   + adblock_status  : enabled
-  + adblock_version : 4.0.2
-  + blocked_domains : 52420
-  + active_sources  : adaway adguard andryou bitcoin disconnect winspy yoyo 
-  + dns_backend     : kresd, /etc/kresd
-  + run_utils       : /usr/bin/curl, /bin/awk
+  + adblock_version : 4.1.0
+  + blocked_domains : 32658
+  + active_sources  : android_tracking, disconnect, shallalist
+  + dns_backend     : dnsmasq, /tmp/dnsmasq.d
+  + run_utils       : /usr/bin/curl, /usr/bin/gawk
   + run_ifaces      : trigger: trm_wwan, report: br-lan
-  + run_directories : base: /tmp, backup: /tmp, report: /tmp, jail: /tmp
-  + run_flags       : backup: 1, reset: 0, flush: 0, force: 1, search: 0, report: 1, mail: 0, jail: 0
-  + last_run        : start, 0m 17s, 496/198/218, 03.04.2020 08:55:14
-  + system          : CZ.NIC Turris Mox Board, TurrisOS 5.1.0 81264ebb51991aa2d17489852854e3b5ec3f514d
+  + run_directories : base: /tmp, backup: /tmp/adblock-Backup, report: /tmp/adblock-Report, jail: /tmp
+  + run_flags       : backup: 1, flush: 0, force: 1, search: 1, report: 1, mail: 0, jail: 0
+  + last_run        : reload, 0m 25s, 252/177/167, 26.02.2021 18:31:41
+  + system          : GL.iNet GL-MT1300, OpenWrt SNAPSHOT r15875-1bf6d70e60
The 'last\_run' line includes the used start type, the run duration, the memory footprint after DNS backend loading (total/free/available) and the date/time of the last run. diff --git a/net/adblock/files/adblock.categories b/net/adblock/files/adblock.categories new file mode 100644 index 000000000..68104c0ed --- /dev/null +++ b/net/adblock/files/adblock.categories @@ -0,0 +1,145 @@ +eng;basic;basic/formats/domains.txt +eng;blu;blu/formats/domains.txt +eng;blugo;bluGo/formats/domains.txt +eng;ext-extreme;extensions/xtreme/formats/domains.txt +eng;ext-pornlite;extensions/porn-lite/formats/domains.txt +eng;ext-regional;extensions/regional/formats/domains.txt +eng;ext-social;extensions/social/formats/domains.txt +eng;porn;porn/formats/domains.txt +eng;spark;spark/formats/domains.txt +eng;ultimate;ultimate/formats/domains.txt +eng;unified;unified/formats/domains.txt +sha;adv +sha;aggressive +sha;alcohol +sha;anonvpn +sha;automobile +sha;chat +sha;costtraps +sha;dating +sha;downloads +sha;drugs +sha;dynamic +sha;education +sha;finance +sha;fortunetelling +sha;forum +sha;gamble +sha;government +sha;hacking +sha;hobby +sha;homestyle +sha;hospitals +sha;imagehosting +sha;isp +sha;jobsearch +sha;library +sha;military +sha;models +sha;movies +sha;music +sha;news +sha;podcasts +sha;politics +sha;porn +sha;radiotv +sha;recreation +sha;redirector +sha;religion +sha;remotecontrol +sha;ringtones +sha;science +sha;searchengines +sha;sex +sha;shopping +sha;socialnet +sha;spyware +sha;tracker +sha;updatesites +sha;urlshortener +sha;violence +sha;warez +sha;weapons +sha;webmail +sha;webphone +sha;webradio +sha;webtv +stb;fakenews;alternates/fakenews/hosts +stb;fakenews-gambling;alternates/fakenews-gambling/hosts +stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts +stb;fakenews-gambling-porn-social;alternates/fakenews-porn-social/hosts +stb;fakenews-gambling-social;alternates/fakenews-gambling-social/hosts +stb;fakenews-porn;alternates/fakenews-porn/hosts +stb;fakenews-porn-social;alternates/fakenews-porn-social/hosts +stb;fakenews-social;alternates/fakenews-social/hosts +stb;gambling;alternates/gambling/hosts +stb;gambling-porn;alternates/gambling-porn/hosts +stb;gambling-porn-social;alternates/gambling-porn-social/hosts +stb;gambling-social;alternates/gambling-social/hosts +stb;porn;alternates/porn/hosts +stb;porn-social;alternates/porn-social/hosts +stb;social;alternates/social/hosts +stb;standard;hosts +utc;adult +utc;agressif +utc;arjel +utc;associations_religieuses +utc;astrology +utc;audio-video +utc;bank +utc;bitcoin +utc;blog +utc;celebrity +utc;chat +utc;child +utc;cleaning +utc;cooking +utc;cryptojacking +utc;dangerous_material +utc;dating +utc;ddos +utc;dialer +utc;doh +utc;download +utc;drogue +utc;educational_games +utc;examen_pix +utc;exceptions_liste_bu +utc;filehosting +utc;financial +utc;forums +utc;gambling +utc;games +utc;hacking +utc;jobsearch +utc;lingerie +utc;liste_blanche +utc;liste_bu +utc;malware +utc;manga +utc;marketingware +utc;mixed_adult +utc;mobile-phone +utc;phishing +utc;press +utc;publicite +utc;radio +utc;reaffected +utc;redirector +utc;remote-control +utc;sect +utc;sexual_education +utc;shopping +utc;shortener +utc;social_networks +utc;special +utc;sports +utc;stalkerware +utc;strict_redirector +utc;strong_redirector +utc;translation +utc;tricheur +utc;update +utc;vpn +utc;warez +utc;webmail diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index 0aa219d44..4efb7d46d 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -1,5 +1,5 @@ #!/bin/sh /etc/rc.common -# Copyright (c) 2015-2020 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2021 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # set (s)hellcheck exceptions @@ -8,13 +8,26 @@ START=30 USE_PROCD=1 -extra_command "suspend" "Suspend adblock processing" -extra_command "resume" "Resume adblock processing" -extra_command "query" " Query active blocklists and backups for a specific domain" -extra_command "report" "[] Print DNS statistics with an optional search parameter" -extra_command "list" "[[|] [source(s)]] List available adblock sources or add/remove them from config" -extra_command "timer" " [] [] Set a cron based update interval" -extra_command "version" "Print version information" +if [ -n "$(type -t extra_command)" ] +then + extra_command "suspend" "Suspend adblock processing" + extra_command "resume" "Resume adblock processing" + extra_command "query" " Query active blocklists and backups for a specific domain" + extra_command "report" "[] Print DNS statistics with an optional search parameter" + extra_command "list" "[|||||||||] List/Edit available sources" + extra_command "timer" "[ [] []]|[ ] List/Edit cron update intervals" + extra_command "version" "Print version information" +else + EXTRA_COMMANDS="status suspend resume query report list timer version" + EXTRA_HELP=" status Service status + suspend Suspend adblock processing + resume Resume adblock processing + query Query active blocklists and backups for a specific domain + report [] Print DNS statistics with an optional search parameter + list [|||||||||] List/Edit available sources + timer [ [] []]|[ ] List/Edit cron update intervals + version Print version information" +fi adb_init="/etc/init.d/adblock" adb_script="/usr/bin/adblock.sh" @@ -97,27 +110,85 @@ report() list() { - local src_archive src_file src_enabled enabled name action="${1}" + local src_archive src_file src_enabled enabled name sha_list utc_list action="${1}" - if [ "${action}" = "add" ] || [ "${action}" = "remove" ] + if [ "${action%_*}" = "add" ] || [ "${action%_*}" = "remove" ] then shift for name in "${@}" do - if [ "${action}" = "add" ] - then - if [ -z "$(uci_get adblock global adb_sources | grep -Fo "${name}")" ] - then - uci_add_list adblock global adb_sources "${name}" - printf "%s\\n" "::: adblock source '${name}' added to config" - fi - else - if [ -n "$(uci_get adblock global adb_sources | grep -Fo "${name}")" ] - then - uci_remove_list adblock global adb_sources "${name}" - printf "%s\\n" "::: adblock source '${name}' removed from config" - fi - fi + case "${action}" in + "add") + if [ -z "$(uci_get adblock global adb_sources | grep -Fo "${name}")" ] + then + uci_add_list adblock global adb_sources "${name}" + printf "%s\n" "::: adblock source '${name}' added to config" + fi + ;; + "remove") + if [ -n "$(uci_get adblock global adb_sources | grep -Fo "${name}")" ] + then + uci_remove_list adblock global adb_sources "${name}" + printf "%s\n" "::: adblock source '${name}' removed from config" + fi + ;; + "add_sha") + if [ -z "$(uci_get adblock global adb_sha_sources | grep -Fo "${name}")" ] + then + uci_add_list adblock global adb_sha_sources "${name}" + printf "%s\n" "::: adblock shallalist '${name}' added to config" + fi + ;; + "remove_sha") + if [ -n "$(uci_get adblock global adb_sha_sources | grep -Fo "${name}")" ] + then + uci_remove_list adblock global adb_sha_sources "${name}" + printf "%s\n" "::: adblock shallalist '${name}' removed from config" + fi + ;; + "add_utc") + if [ -z "$(uci_get adblock global adb_utc_sources | grep -Fo "${name}")" ] + then + uci_add_list adblock global adb_utc_sources "${name}" + printf "%s\n" "::: adblock utcapitole '${name}' added to config" + fi + ;; + "remove_utc") + if [ -n "$(uci_get adblock global adb_utc_sources | grep -Fo "${name}")" ] + then + uci_remove_list adblock global adb_utc_sources "${name}" + printf "%s\n" "::: adblock utcapitole '${name}' removed from config" + fi + ;; + "add_eng") + if [ -z "$(uci_get adblock global adb_eng_sources | grep -Fo "${name}")" ] + then + uci_add_list adblock global adb_eng_sources "${name}" + printf "%s\n" "::: adblock energized '${name}' added to config" + fi + ;; + "remove_eng") + if [ -n "$(uci_get adblock global adb_eng_sources | grep -Fo "${name}")" ] + then + uci_remove_list adblock global adb_eng_sources "${name}" + printf "%s\n" "::: adblock energized '${name}' removed from config" + fi + ;; + "add_stb") + if [ -z "$(uci_get adblock global adb_stb_sources | grep -Fo "${name}")" ] + then + uci_add_list adblock global adb_stb_sources "${name}" + printf "%s\n" "::: adblock stevenblack '${name}' added to config" + fi + ;; + "remove_stb") + if [ -n "$(uci_get adblock global adb_stb_sources | grep -Fo "${name}")" ] + then + uci_remove_list adblock global adb_stb_sources "${name}" + printf "%s\n" "::: adblock stevenblack '${name}' removed from config" + fi + ;; + esac done if [ -n "$(uci -q changes adblock)" ] then @@ -132,16 +203,16 @@ list() then zcat "${src_archive}" > "${src_file}" else - printf "%s\\n" "::: adblock source archive '${src_archive}' not found" + printf "%s\n" "::: adblock source archive '${src_archive}' not found" fi if [ -r "${src_file}" ] then src_enabled="${src_enabled#*=}" src_enabled="${src_enabled//\'}" - printf "%s\\n" "::: Available adblock sources" - printf "%s\\n" ":::" - printf "%-25s%-10s%-7s%-21s%s\\n" " Name" "Enabled" "Size" "Focus" "Info URL" - printf "%s\\n" " -------------------------------------------------------------------" + printf "%s\n" "::: Available adblock sources" + printf "%s\n" ":::" + printf "%-25s%-10s%-7s%-21s%s\n" " Name" "Enabled" "Size" "Focus" "Info URL" + printf "%s\n" " -------------------------------------------------------------------" json_load_file "${src_file}" json_get_keys keylist for key in ${keylist} @@ -161,98 +232,140 @@ list() enabled=" " fi src_enabled="${src_enabled/${key}}" - printf " + %-21s%-10s%-7s%-21s%s\\n" "${key:0:20}" "${enabled}" "${size:0:3}" "${focus:0:20}" "${descurl:0:50}" + printf " + %-21s%-10s%-7s%-21s%s\n" "${key:0:20}" "${enabled}" "${size:0:3}" "${focus:0:20}" "${descurl:0:50}" else src_enabled="${src_enabled} ${key}" fi json_select .. done + sha_list="$(uci_get adblock global adb_sha_sources "-")" + utc_list="$(uci_get adblock global adb_utc_sources "-")" + eng_list="$(uci_get adblock global adb_eng_sources "-")" + stb_list="$(uci_get adblock global adb_stb_sources "-")" + printf "%s\n" " ---------------------------------------------------------------------------" + printf " * %s\n" "Configured shallalist categories: ${sha_list// /, }" + printf " * %s\n" "Configured utcapitole categories: ${utc_list// /, }" + printf " * %s\n" "Configured energized variants: ${eng_list// /, }" + printf " * %s\n" "Configured stevenblack variants: ${stb_list// /, }" + if [ -n "${src_enabled// }" ] then - printf "%s\\n" " ----------------------------------------------" - printf "%s\\n" " Sources without valid configuration" - printf "%s\\n" " ----------------------------------------------" + printf "%s\n" " ---------------------------------------------------------------------------" + printf "%s\n" " Sources without valid configuration" + printf "%s\n" " ---------------------------------------------------------------------------" for key in ${src_enabled} do - printf " - %s\\n" "${key:0:20}" + printf " - %s\n" "${key:0:20}" done fi else - printf "%s\\n" "::: adblock source file '${src_file}' not found" + printf "%s\n" "::: adblock source file '${src_file}' not found" fi fi } +status() +{ + status_service +} + status_service() { - local key keylist value rtfile + local key keylist value idxval values rtfile rtfile="$(uci_get adblock global adb_rtfile "/tmp/adb_runtime.json")" - if [ -s "${rtfile}" ] + + json_load_file "${rtfile}" >/dev/null 2>&1 + json_get_keys keylist + if [ -n "${keylist}" ] then - printf "%s\\n" "::: adblock runtime information" - json_load_file "${rtfile}" - json_select data - json_get_keys keylist + printf "%s\n" "::: adblock runtime information" for key in ${keylist} do - json_get_var value "${key}" - if [ "${key}" = "active_sources" ] + json_get_var value "${key}" >/dev/null 2>&1 + if [ "${key%_*}" = "active" ] then printf " + %-15s : " "${key}" - json_select "${key}" + json_select "${key}" >/dev/null 2>&1 + values="" index=1 - while json_get_type status "${index}" && [ "${status}" = "object" ] + while json_get_type type "${index}" && [ "${type}" = "object" ] do - json_get_values source "${index}" - printf "%s " "${source}" + json_get_values idxval "${index}" >/dev/null 2>&1 + if [ "${index}" = "1" ] + then + values="${idxval}" + else + values="${values}, ${idxval}" + fi index=$((index+1)) done - printf "\\n" + values="$(printf "%s" "${values}" | awk '{NR=1;max=98;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-22s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" + printf "%s\n" "${values:-"-"}" json_select ".." else - printf " + %-15s : %s\\n" "${key}" "${value}" + printf " + %-15s : %s\n" "${key}" "${value:-"-"}" fi done else - printf "%s\\n" "::: no adblock runtime information available" + printf "%s\n" "::: no adblock runtime information available" fi } timer() { - local action="${1}" hour="${2}" minute="${3:-0}" weekday="${4:-"*"}" + local cron_file cron_content cron_lineno action="${1:-"list"}" cron_tasks="${2}" hour="${3}" minute="${4:-0}" weekday="${5:-"*"}" + + cron_file="/etc/crontabs/root" - hour="${hour//[[:alpha:]]/}" - minute="${minute//[[:alpha:]]/}" - if [ -n "${action}" ] && [ -n "${hour}" ] && [ -n "${minute}" ] && [ -n "${weekday}" ] && \ - [ "${hour}" -ge 0 ] && [ "${hour}" -le 23 ] && \ - [ "${minute}" -ge 0 ] && [ "${minute}" -le 59 ] + if [ -s "${cron_file}" ] && [ "${action}" = "list" ] + then + awk '{print NR "> " $0}' "${cron_file}" + elif [ "${action}" = "add" ] then - if [ -r "/etc/crontabs/root" ] + hour="${hour//[[:alpha:]]/}" + minute="${minute//[[:alpha:]]/}" + if [ -n "${cron_tasks}" ] && [ -n "${hour}" ] && [ -n "${minute}" ] && [ -n "${weekday}" ] && \ + [ "${hour}" -ge 0 ] && [ "${hour}" -le 23 ] && \ + [ "${minute}" -ge 0 ] && [ "${minute}" -le 59 ] then - search="${adb_init//\//\\/}" - search="${search//./\\.}" - sed -i "/${search}/d" "/etc/crontabs/root" + printf "%02d %02d %s\n" "${minute}" "${hour}" "* * ${weekday} ${adb_init} ${cron_tasks}" >> "${cron_file}" + /etc/init.d/cron restart + fi + elif [ -s "${cron_file}" ] && [ "${action}" = "remove" ] + then + cron_tasks="${cron_tasks//[[:alpha:]]/}" + cron_lineno="$(awk 'END{print NR}' "${cron_file}")" + cron_content="$(awk '{print $0}' "${cron_file}")" + if [ "${cron_tasks:-"0"}" -le "${cron_lineno:-"1"}" ] && [ -n "${cron_content}" ] + then + printf "%s\n" "${cron_content}" | awk "NR!~/^${cron_tasks}$/" > "${cron_file}" + /etc/init.d/cron restart fi - printf "%02d %02d %s\\n" "${minute}" "${hour}" "* * ${weekday} ${adb_init} ${action}" >> "/etc/crontabs/root" - /etc/init.d/cron restart - else - printf "%s\\n" "::: the refresh timer could not been updated" fi } service_triggers() { - local trigger delay type + local iface delay - trigger="$(uci_get adblock global adb_trigger)" - delay="$(uci_get adblock global adb_triggerdelay "2")" - type="$(uci_get adblock global adb_starttype "start")" + iface="$(uci_get adblock global adb_trigger)" + delay="$(uci_get adblock global adb_triggerdelay "5")" PROCD_RELOAD_DELAY=$((delay*1000)) - if [ -n "${trigger}" ] + + if [ -z "${iface}" ] + then + . "/lib/functions/network.sh" + network_find_wan iface + if [ -n "${iface}" ] + then + uci_set adblock global adb_trigger "${iface}" + uci_commit "adblock" + fi + fi + if [ -n "${iface}" ] then - procd_add_interface_trigger "interface.*.up" "${trigger}" "${adb_init}" "${type}" + procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" "start" fi procd_add_reload_trigger "adblock" } diff --git a/net/adblock/files/adblock.monitor b/net/adblock/files/adblock.monitor deleted file mode 100755 index b5b73673d..000000000 --- a/net/adblock/files/adblock.monitor +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# ubus monitor to trace dns backend events and conditionally restart adblock -# Copyright (c) 2015-2020 Dirk Brenken (dev@brenken.org) -# This is free software, licensed under the GNU General Public License v3. - -# set (s)hellcheck exceptions -# shellcheck disable=1091,2010,2016,2034,2039,2059,2086,2091,2129,2143,2154,2181,2183,2188 - -LC_ALL=C -PATH="/usr/sbin:/usr/bin:/sbin:/bin" - -if [ -r "/lib/functions.sh" ] -then - . "/lib/functions.sh" - adb_dns="$(uci_get adblock global adb_dns)" -fi -adb_ver="${1}" -adb_ubus="$(command -v ubus)" -adb_logger="$(command -v logger)" - -f_log() -{ - local class="${1}" log_msg="${2}" - - if [ -x "${adb_logger}" ] - then - "${adb_logger}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" - else - printf "%s %s %s\\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - fi -} - -if [ -x "${adb_ubus}" ] && [ -n "${adb_dns}" ] -then - f_log "info" "ubus/adblock monitor started" - "${adb_ubus}" -S -M r -m invoke monitor | \ - { grep -qE "\"method\":\"(set|signal)\",\"data\":\{\"name\":\"${adb_dns}\""; [ $? -eq 0 ] && /etc/init.d/adblock start; } -else - f_log "err" "can't start ubus/adblock monitor" -fi diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index 944cf27f0..d987e0aa6 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -1,6 +1,6 @@ #!/bin/sh # dns based ad/abuse domain blocking -# Copyright (c) 2015-2020 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2021 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # set (s)hellcheck exceptions @@ -11,12 +11,11 @@ export LC_ALL=C export PATH="/usr/sbin:/usr/bin:/sbin:/bin" set -o pipefail -adb_ver="4.0.8" +adb_ver="4.1.0" adb_enabled=0 adb_debug=0 adb_forcedns=0 adb_maxqueue=4 -adb_dnsfilereset=0 adb_dnsflush=0 adb_dnstimeout=20 adb_safesearch=0 @@ -33,13 +32,12 @@ adb_dns="" adb_dnsprefix="adb_list" adb_locallist="blacklist whitelist" adb_tmpbase="/tmp" -adb_backupdir="/tmp" -adb_reportdir="/tmp" +adb_backupdir="${adb_tmpbase}/adblock-Backup" +adb_reportdir="${adb_tmpbase}/adblock-Report" adb_jaildir="/tmp" adb_pidfile="/var/run/adblock.pid" adb_blacklist="/etc/adblock/adblock.blacklist" adb_whitelist="/etc/adblock/adblock.whitelist" -adb_ubusservice="/etc/adblock/adblock.monitor" adb_mailservice="/etc/adblock/adblock.mail" adb_dnsfile="${adb_dnsprefix}.overall" adb_dnsjail="${adb_dnsprefix}.jail" @@ -75,12 +73,11 @@ f_load() f_fetch fi - if [ "${adb_enabled}" -eq 0 ] + if [ "${adb_enabled}" = "0" ] then f_extconf f_temp f_rmdns - f_bgserv "stop" f_jsnup "disabled" f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" exit 0 @@ -97,7 +94,7 @@ f_env() f_extconf f_temp - if [ "${adb_dnsflush}" -eq 1 ] + if [ "${adb_dnsflush}" = "1" ] then printf "${adb_dnsheader}" > "${adb_dnsdir}/${adb_dnsfile}" f_dnsup @@ -146,6 +143,18 @@ f_conf() if [ "${option}" = "adb_sources" ] then eval "${option}=\"$(printf "%s" "${adb_sources}") ${value}\"" + elif [ "${option}" = "adb_eng_sources" ] + then + eval "${option}=\"$(printf "%s" "${adb_eng_sources}") ${value}\"" + elif [ "${option}" = "adb_sha_sources" ] + then + eval "${option}=\"$(printf "%s" "${adb_sha_sources}") ${value}\"" + elif [ "${option}" = "adb_stb_sources" ] + then + eval "${option}=\"$(printf "%s" "${adb_stb_sources}") ${value}\"" + elif [ "${option}" = "adb_utc_sources" ] + then + eval "${option}=\"$(printf "%s" "${adb_utc_sources}") ${value}\"" elif [ "${option}" = "adb_safesearchlist" ] then eval "${option}=\"$(printf "%s" "${adb_safesearchlist}") ${value}\"" @@ -299,7 +308,7 @@ f_dns() if [ -n "${adb_dnsdir}" ] && [ ! -d "${adb_dnsdir}" ] then mkdir -p "${adb_dnsdir}" - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then f_log "info" "dns backend directory '${adb_dnsdir}' created" else @@ -315,27 +324,27 @@ f_dns() if [ "${dns_up}" != "true" ] then f_dnsup 4 - if [ "${?}" -ne 0 ] + if [ "${?}" != "0" ] then f_log "err" "dns backend '${adb_dns}' not running or executable" fi fi - if [ "${adb_backup}" -eq 1 ] && [ -n "${adb_backupdir}" ] && [ ! -d "${adb_backupdir}" ] + if [ "${adb_backup}" = "1" ] && [ -n "${adb_backupdir}" ] && [ ! -d "${adb_backupdir}" ] then mkdir -p "${adb_backupdir}" - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then f_log "info" "backup directory '${adb_backupdir}' created" else - f_log "err" "backup backend directory '${adb_backupdir}' could not be created" + f_log "err" "backup directory '${adb_backupdir}' could not be created" fi fi if [ -n "${adb_jaildir}" ] && [ ! -d "${adb_jaildir}" ] then mkdir -p "${adb_jaildir}" - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then f_log "info" "jail directory '${adb_jaildir}' created" else @@ -405,11 +414,11 @@ f_temp() cpu="$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)" core="$(grep -cm1 '^core id' /proc/cpuinfo 2>/dev/null)" - if [ "${cpu}" -eq 0 ] + if [ "${cpu}" = "0" ] then cpu=1 fi - if [ "${core}" -eq 0 ] + if [ "${core}" = "0" ] then core=1 fi @@ -454,7 +463,7 @@ f_rmdns() if [ "${adb_dns}" = "raw" ] || { [ -n "${adb_dns}" ] && [ -n "${status}" ]; } then > "${adb_rtfile}" - if [ "${adb_backup}" -eq 1 ] + if [ "${adb_backup}" = "1" ] then rm -f "${adb_backupdir}/${adb_dnsprefix}".*.gz fi @@ -566,7 +575,7 @@ f_extconf() "dnsmasq") config="dhcp" config_dir="$(uci_get dhcp "@dnsmasq[${adb_dnsinstance}]" confdir | grep -Fo "${adb_dnsdir}")" - if [ "${adb_enabled}" -eq 1 ] && [ -z "${config_dir}" ] + if [ "${adb_enabled}" = "1" ] && [ -z "${config_dir}" ] then uci_set dhcp "@dnsmasq[${adb_dnsinstance}]" confdir "${adb_dnsdir}" 2>/dev/null fi @@ -574,10 +583,10 @@ f_extconf() "kresd") config="resolver" config_file="$(uci_get resolver kresd rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" - if [ "${adb_enabled}" -eq 1 ] && [ -z "${config_file}" ] + if [ "${adb_enabled}" = "1" ] && [ -z "${config_file}" ] then uci -q add_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - elif [ "${adb_enabled}" -eq 0 ] && [ -n "${config_file}" ] + elif [ "${adb_enabled}" = "0" ] && [ -n "${config_file}" ] then uci -q del_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" fi @@ -587,8 +596,8 @@ f_extconf() config="firewall" fwcfg="$(uci -qNX show "${config}" | "${adb_awk}" 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" - if [ "${adb_enabled}" -eq 1 ] && [ "${adb_forcedns}" -eq 1 ] && \ - [ "$(/etc/init.d/firewall enabled; printf "%u" ${?})" -eq 0 ] + if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && \ + [ "$(/etc/init.d/firewall enabled; printf "%u" ${?})" = "0" ] then for zone in ${adb_zonelist} do @@ -612,7 +621,7 @@ f_extconf() fwcfg="${fwcfg#"${fwcfg%%[![:space:]]*}"}" fwcfg="${fwcfg%"${fwcfg##*[![:space:]]}"}" fi - if [ "${adb_enabled}" -eq 0 ] || [ "${adb_forcedns}" -eq 0 ] || [ -n "${fwcfg}" ] + if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ] || [ -n "${fwcfg}" ] then for section in ${fwcfg} do @@ -628,7 +637,7 @@ f_dnsup() { local dns_service dns_up dns_pid dns_procfile restart_rc cnt=0 out_rc=4 in_rc="${1:-0}" - if [ "${adb_dns}" = "raw" ] || { [ "${in_rc}" -eq 0 ] && [ "${adb_dnsinotify}" -eq 1 ]; } + if [ "${adb_dns}" = "raw" ] || { [ "${in_rc}" = "0" ] && [ "${adb_dnsinotify}" = "1" ]; } then out_rc=0 else @@ -648,7 +657,7 @@ f_dnsup() if [ -x "${adb_lookupcmd}" ] && [ "${adb_lookupdomain}" != "false" ] then "${adb_lookupcmd}" "${adb_lookupdomain}" >/dev/null 2>&1 - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then out_rc=0 break @@ -690,16 +699,16 @@ f_list() rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" "${adb_awk}" "${rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.raw.${src_name}" out_rc="${?}" - if [ "${out_rc}" -eq 0 ] + if [ "${out_rc}" = "0" ] then rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\\.\",\"\\\\.\",\$1);print tolower(\"^(|.*\\\\.)\"\$1\"$\")}" "${adb_awk}" "${rset}" "${adb_tmpdir}/tmp.raw.${src_name}" > "${adb_tmpdir}/tmp.rem.${src_name}" out_rc="${?}" - if [ "${out_rc}" -eq 0 ] && [ "${adb_dnsallow}" != "1" ] + if [ "${out_rc}" = "0" ] && [ "${adb_dnsallow}" != "1" ] then eval "${adb_dnsallow}" "${adb_tmpdir}/tmp.raw.${src_name}" > "${adb_tmpdir}/tmp.add.${src_name}" out_rc="${?}" - if [ "${out_rc}" -eq 0 ] && [ "${adb_jail}" = "1" ] && [ "${adb_dnsstop}" != "0" ] + if [ "${out_rc}" = "0" ] && [ "${adb_jail}" = "1" ] && [ "${adb_dnsstop}" != "0" ] then > "${adb_jaildir}/${adb_dnsjail}" if [ -n "${adb_dnsheader}" ] @@ -720,19 +729,19 @@ f_list() safe_url="https://www.google.com/supported_domains" safe_cname="forcesafesearch.google.com" safe_domains="${adb_tmpdir}/tmp.load.safesearch.${src_name}" - if [ "${adb_backup}" -eq 1 ] && [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ] + if [ "${adb_backup}" = "1" ] && [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ] then zcat "${adb_backupdir}/safesearch.${src_name}.gz" > "${safe_domains}" out_rc="${?}" else "${adb_fetchutil}" ${adb_fetchparm} "${safe_domains}" "${safe_url}" 2>/dev/null out_rc="${?}" - if [ "${adb_backup}" -eq 1 ] && [ "${out_rc}" -eq 0 ] + if [ "${adb_backup}" = "1" ] && [ "${out_rc}" = "0" ] then gzip -cf "${safe_domains}" > "${adb_backupdir}/safesearch.${src_name}.gz" fi fi - if [ "${out_rc}" -eq 0 ] + if [ "${out_rc}" = "0" ] then if [ -x "${adb_lookupcmd}" ] then @@ -786,7 +795,7 @@ f_list() ;; "yandex") safe_cname="familysearch.yandex.ru" - safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az" + safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" if [ -x "${adb_lookupcmd}" ] then safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" @@ -798,7 +807,7 @@ f_list() out_rc="${?}" ;; "youtube") - if [ "${adb_safesearchmod}" -eq 0 ] + if [ "${adb_safesearchmod}" = "0" ] then safe_cname="restrict.youtube.com" else @@ -816,7 +825,7 @@ f_list() out_rc="${?}" ;; esac - if [ "${out_rc}" -eq 0 ] && [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ] + if [ "${out_rc}" = "0" ] && [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ] then > "${adb_tmpdir}/tmp.safesearch.${src_name}" if [ "${adb_dns}" = "named" ] @@ -828,7 +837,7 @@ f_list() for item in ${array} do eval "${adb_dnssafesearch}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >> "${adb_tmpdir}/tmp.safesearch.${src_name}" - if [ "${?}" -ne 0 ] + if [ "${?}" != "0" ] then rm -f "${adb_tmpdir}/tmp.safesearch.${src_name}" break @@ -859,7 +868,7 @@ f_list() name="${name%.*}" zcat "${file}" > "${adb_tmpfile}.${name}" & hold=$((cnt%adb_maxqueue)) - if [ "${hold}" -eq 0 ] + if [ "${hold}" = "0" ] then wait fi @@ -871,13 +880,13 @@ f_list() else out_rc=4 fi - if [ "${adb_action}" != "start" ] && [ "${adb_action}" != "resume" ] && [ -n "${src_name}" ] && [ "${out_rc}" -ne 0 ] + if [ "${adb_action}" != "start" ] && [ "${adb_action}" != "resume" ] && [ -n "${src_name}" ] && [ "${out_rc}" != "0" ] then adb_sources="${adb_sources/${src_name}}" fi ;; "remove") - if [ "${adb_backup}" -eq 1 ] + if [ "${adb_backup}" = "1" ] then rm "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null fi @@ -885,13 +894,13 @@ f_list() adb_sources="${adb_sources/${src_name}}" ;; "merge") - if [ "${adb_backup}" -eq 1 ] + if [ "${adb_backup}" = "1" ] then for src_name in ${adb_sources} do ffiles="${ffiles} -a ! -name ${adb_dnsprefix}.${src_name}.gz" done - if [ "${adb_safesearch}" -eq 1 ] && [ "${adb_dnssafesearch}" != "0" ] + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ] then ffiles="${ffiles} -a ! -name safesearch.google.gz" fi @@ -944,7 +953,7 @@ f_tld() "${adb_awk}" '{if(NR==1){tld=$NF};while(getline){if(index($NF,tld".")==0){print tld;tld=$NF}}print tld}' "${source}" | \ "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' > "${temp_tld}" - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then mv -f "${temp_tld}" "${source}" cnt_tld="$(wc -l 2>/dev/null < "${source}")" @@ -979,13 +988,8 @@ f_switch() then if [ "${mode}" = "suspend" ] then - f_bgserv "stop" f_dnsup fi - if [ "${mode}" = "resume" ] - then - f_bgserv "start" - fi f_jsnup "${mode}" f_log "info" "${mode} adblock processing" fi @@ -1000,7 +1004,7 @@ f_query() if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then - printf "%s\\n" "::: invalid input, please submit a single (sub-)domain :::" + printf "%s\n" "::: invalid input, please submit a single (sub-)domain :::" else case "${adb_dns}" in "dnsmasq") @@ -1030,48 +1034,45 @@ f_query() ;; esac query_start="$(date "+%s")" - if [ "${adb_dnsfilereset}" -eq 0 ] - then - while [ "${domain}" != "${tld}" ] - do - search="${domain//[+*~%\$&\"\']/}" - search="${search//./\\.}" - result="$("${adb_awk}" -F '/|\"|\t| ' "/^(${search}|${prefix}+${search}.*${suffix})$/{i++;if(i<=9){printf \" + %s\\n\",\$${field}}else if(i==10){printf \" + %s\\n\",\"[...]\";exit}}" "${adb_dnsdir}/${adb_dnsfile}")" - printf "%s\\n%s\\n%s\\n" ":::" "::: domain '${domain}' in active blocklist" ":::" - printf "%s\\n\\n" "${result:-" - no match"}" - domain="${tld}" - tld="${domain#*.}" - done - fi - if [ "${adb_backup}" -eq 1 ] && [ -d "${adb_backupdir}" ] + while [ "${domain}" != "${tld}" ] + do + search="${domain//[+*~%\$&\"\']/}" + search="${search//./\\.}" + result="$("${adb_awk}" -F '/|\"|\t| ' "/^(${search}|${prefix}+${search}.*${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_dnsdir}/${adb_dnsfile}")" + printf "%s\n%s\n%s\n" ":::" "::: domain '${domain}' in active blocklist" ":::" + printf "%s\n\n" "${result:-" - no match"}" + domain="${tld}" + tld="${domain#*.}" + done + if [ "${adb_backup}" = "1" ] && [ -d "${adb_backupdir}" ] then search="${1//[+*~%\$&\"\']/}" search="${search//./\\.}" - printf "%s\\n%s\\n%s\\n" ":::" "::: domain '${1}' in backups and black-/whitelist" ":::" + printf "%s\n%s\n%s\n" ":::" "::: domain '${1}' in backups and black-/whitelist" ":::" for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz "${adb_blacklist}" "${adb_whitelist}" do suffix="${file##*.}" if [ "${suffix}" = "gz" ] then zcat "${file}" 2>/dev/null | \ - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\\n\",f,\"[...]\"}};END{exit rc}" + "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" else - "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\\n\",f,\"[...]\"}};END{exit rc}" "${file}" + "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" fi - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then result="true" query_end="$(date "+%s")" if [ "$((query_end-query_start))" -gt "${query_timeout}" ] then - printf "%s\\n\\n" " - [...]" + printf "%s\n\n" " - [...]" break fi fi done if [ "${result}" != "true" ] then - printf "%s\\n\\n" " - no match" + printf "%s\n\n" " - no match" fi fi fi @@ -1106,8 +1107,7 @@ f_jsnup() ;; esac json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_select "data" >/dev/null 2>&1 - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then if [ -z "${adb_fetchutil}" ] || [ -z "${adb_awk}" ] then @@ -1128,7 +1128,6 @@ f_jsnup() > "${adb_rtfile}" json_load_file "${adb_rtfile}" >/dev/null 2>&1 json_init - json_add_object "data" json_add_string "adblock_status" "${status:-"enabled"}" json_add_string "adblock_version" "${adb_ver}" json_add_string "blocked_domains" "${adb_cnt:-0}" @@ -1144,13 +1143,12 @@ f_jsnup() json_add_string "run_utils" "${utils:-"-"}" json_add_string "run_ifaces" "trigger: ${adb_trigger:-"-"}, report: ${adb_repiface:-"-"}" json_add_string "run_directories" "base: ${adb_tmpbase}, backup: ${adb_backupdir}, report: ${adb_reportdir}, jail: ${adb_jaildir}" - json_add_string "run_flags" "backup: ${adb_backup}, reset: ${adb_dnsfilereset}, flush: ${adb_dnsflush}, force: ${adb_forcedns}, search: ${adb_safesearch}, report: ${adb_report}, mail: ${adb_mail}, jail: ${adb_jail}" + json_add_string "run_flags" "backup: ${adb_backup}, flush: ${adb_dnsflush}, force: ${adb_forcedns}, search: ${adb_safesearch}, report: ${adb_report}, mail: ${adb_mail}, jail: ${adb_jail}" json_add_string "last_run" "${runtime:-"-"}" json_add_string "system" "${adb_sysver}" - json_close_object json_dump > "${adb_rtfile}" - if [ "${adb_mail}" -eq 1 ] && [ -x "${adb_mailservice}" ] && \ + if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && \ { [ "${status}" = "error" ] || { [ "${status}" = "enabled" ] && [ "${adb_cnt}" -le "${adb_mailcnt}" ]; } } then ( "${adb_mailservice}" "${adb_ver}" >/dev/null 2>&1 )& @@ -1165,42 +1163,23 @@ f_log() { local class="${1}" log_msg="${2}" - if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${adb_debug}" -eq 1 ]; } + if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${adb_debug}" = "1" ]; } then if [ -x "${adb_loggercmd}" ] then "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" else - printf "%s %s %s\\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" + printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" fi if [ "${class}" = "err" ] then f_rmdns - f_bgserv "stop" f_jsnup "error" exit 1 fi fi } -# start ubus monitor service to trace dns backend events -# -f_bgserv() -{ - local bg_pid status="${1}" - - bg_pid="$(pgrep -f "^/bin/sh ${adb_ubusservice}.*|^/bin/ubus -S -M r -m invoke monitor|^grep -qF \"method\":\"set\",\"data\":\\{\"name\":\"${adb_dns}\"" | "${adb_awk}" '{ORS=" "; print $1}')" - if [ "${adb_dns}" != "raw" ] && [ -z "${bg_pid}" ] && [ "${status}" = "start" ] \ - && [ -x "${adb_ubusservice}" ] && [ "${adb_dnsfilereset}" -eq 1 ] - then - ( "${adb_ubusservice}" "${adb_ver}" & ) - elif [ -n "${bg_pid}" ] && [ "${status}" = "stop" ] - then - kill -HUP "${bg_pid}" 2>/dev/null - fi - f_log "debug" "f_bgserv ::: status: ${status:-"-"}, bg_pid: ${bg_pid:-"-"}, dns_filereset: ${adb_dnsfilereset:-"-"}, ubus_service: ${adb_ubusservice:-"-"}" -} - # main function for blocklist processing # f_main() @@ -1219,7 +1198,7 @@ f_main() # safe search preparation # - if [ "${adb_safesearch}" -eq 1 ] && [ "${adb_dnssafesearch}" != "0" ] + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ] then if [ -z "${adb_safesearchlist}" ] then @@ -1237,15 +1216,15 @@ f_main() for src_name in ${adb_sources} do json_select "${src_name}" >/dev/null 2>&1 - if [ "${?}" -ne 0 ] + if [ "${?}" != "0" ] then adb_sources="${adb_sources/${src_name}}" continue fi json_get_var src_url "url" >/dev/null 2>&1 json_get_var src_rset "rule" >/dev/null 2>&1 - json_get_values src_cat "categories" >/dev/null 2>&1 json_select .. + src_tmpcat="${adb_tmpload}.${src_name}.cat" src_tmpload="${adb_tmpload}.${src_name}.load" src_tmpsort="${adb_tmpload}.${src_name}.sort" src_tmpfile="${adb_tmpfile}.${src_name}" @@ -1261,10 +1240,10 @@ f_main() # backup mode # - if [ "${adb_backup}" -eq 1 ] && { [ "${adb_action}" = "start" ] || [ "${adb_action}" = "resume" ]; } + if [ "${adb_backup}" = "1" ] && { [ "${adb_action}" = "start" ] || [ "${adb_action}" = "resume" ]; } then f_list restore - if [ "${?}" -eq 0 ] && [ -s "${src_tmpfile}" ] + if [ "${?}" = "0" ] && [ -s "${src_tmpfile}" ] then continue fi @@ -1272,70 +1251,104 @@ f_main() # download queue processing # - if [ -n "${src_cat}" ] + unset src_cat src_entries + if [ "${src_name}" = "shallalist" ] || [ "${src_name}" = "utcapitole" ] then - ( - src_arc="${adb_tmpdir}/${src_url##*/}" - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${src_url}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" -eq 0 ] && [ -s "${src_arc}" ] - then - unset src_entries - src_suffix="$(eval printf "%s" \"\$\{adb_src_suffix_${src_name}:-\"domains\"\}\")" - src_list="$(tar -tzf "${src_arc}" 2>/dev/null)" - for src_item in ${src_cat} - do - src_entries="${src_entries} $(printf "%s" "${src_list}" | grep -E "${src_item}/${src_suffix}$")" - done - if [ -n "${src_entries}" ] - then - tar -xOzf "${src_arc}" ${src_entries} 2>/dev/null > "${src_tmpload}" - src_rc="${?}" - fi - rm -f "${src_arc}" - else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - fi - if [ "${src_rc}" -eq 0 ] && [ -s "${src_tmpload}" ] - then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ] + if [ "${src_name}" = "shallalist" ] && [ -n "${adb_sha_sources}" ] + then + src_cat="${adb_sha_sources}" + elif [ "${src_name}" = "utcapitole" ] && [ -n "${adb_utc_sources}" ] + then + src_cat="${adb_utc_sources}" + fi + if [ -n "${src_cat}" ] + then + ( + src_arc="${adb_tmpdir}/${src_url##*/}" + src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${src_url}" 2>&1)" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_arc}" ] then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | \ - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' > "${src_tmpsort}" + src_suffix="$(eval printf "%s" \"\$\{adb_src_suffix_${src_name}:-\"domains\"\}\")" + src_list="$(tar -tzf "${src_arc}" 2>/dev/null)" + for src_item in ${src_cat} + do + src_entries="${src_entries} $(printf "%s" "${src_list}" | grep -E "${src_item}/${src_suffix}$")" + done + if [ -n "${src_entries}" ] + then + tar -xOzf "${src_arc}" ${src_entries} 2>/dev/null > "${src_tmpload}" + src_rc="${?}" + fi + > "${src_arc}" else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | \ - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' > "${src_tmpsort}" + src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" + f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" fi - rm -f "${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null > "${src_tmpfile}" - src_rc="${?}" - rm -f "${src_tmpsort}" - if [ "${src_rc}" -eq 0 ] && [ -s "${src_tmpfile}" ] + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ] then - f_list download - if [ "${adb_backup}" -eq 1 ] + if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ] then - f_list backup + "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | \ + grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' > "${src_tmpsort}" + else + "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | \ + "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' > "${src_tmpsort}" fi - elif [ "${adb_backup}" -eq 1 ] && [ "${adb_action}" != "start" ] + > "${src_tmpload}" + "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null > "${src_tmpfile}" + src_rc="${?}" + > "${src_tmpsort}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ] + then + f_list download + if [ "${adb_backup}" = "1" ] + then + f_list backup + fi + elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] + then + f_log "info" "archive preparation of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" + f_list restore + rm -f "${src_tmpfile}" + fi + elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] then - f_log "info" "archive preparation of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" + f_log "info" "archive extraction of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" f_list restore - rm -f "${src_tmpfile}" fi - elif [ "${adb_backup}" -eq 1 ] && [ "${adb_action}" != "start" ] - then - f_log "info" "archive extraction of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - fi - )& - continue + )& + fi else + if [ "${src_name}" = "energized" ] && [ -n "${adb_eng_sources}" ] + then + src_cat="${adb_eng_sources}" + elif [ "${src_name}" = "stevenblack" ] && [ -n "${adb_stb_sources}" ] + then + src_cat="${adb_stb_sources}" + elif { [ "${src_name}" = "energized" ] && [ -z "${adb_eng_sources}" ]; } || \ + { [ "${src_name}" = "stevenblack" ] && [ -z "${adb_stb_sources}" ]; } + then + continue + fi ( - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" -eq 0 ] && [ -s "${src_tmpload}" ] + for suffix in ${src_cat:-${src_url}} + do + if [ "${src_url}" != "${suffix}" ] + then + src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>&1)" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ] + then + cat "${src_tmpcat}" >> "${src_tmpload}" + > "${src_tmpcat}" + fi + else + src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>&1)" + src_rc="${?}" + fi + done + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ] then if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ] then @@ -1345,18 +1358,18 @@ f_main() "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | \ "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' > "${src_tmpsort}" fi - rm -f "${src_tmpload}" + > "${src_tmpload}" "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null > "${src_tmpfile}" src_rc="${?}" - rm -f "${src_tmpsort}" - if [ "${src_rc}" -eq 0 ] && [ -s "${src_tmpfile}" ] + > "${src_tmpsort}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ] then f_list download - if [ "${adb_backup}" -eq 1 ] + if [ "${adb_backup}" = "1" ] then f_list backup fi - elif [ "${adb_backup}" -eq 1 ] && [ "${adb_action}" != "start" ] + elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] then f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" f_list restore @@ -1365,7 +1378,7 @@ f_main() else src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - if [ "${adb_backup}" -eq 1 ] && [ "${adb_action}" != "start" ] + if [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] then f_list restore fi @@ -1373,7 +1386,7 @@ f_main() )& fi hold=$((cnt%adb_maxqueue)) - if [ "${hold}" -eq 0 ] + if [ "${hold}" = "0" ] then wait fi @@ -1384,7 +1397,7 @@ f_main() # tld compression and dns restart # - if [ "${?}" -eq 0 ] && [ -s "${adb_tmpdir}/${adb_dnsfile}" ] + if [ "${?}" = "0" ] && [ -s "${adb_tmpdir}/${adb_dnsfile}" ] then f_tld "${adb_tmpdir}/${adb_dnsfile}" f_list final @@ -1393,20 +1406,13 @@ f_main() fi chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null f_dnsup - if [ "${?}" -eq 0 ] + if [ "${?}" = "0" ] then if [ "${adb_action}" != "resume" ] then f_jsnup "enabled" fi - if [ "${adb_dns}" != "raw" ] && [ "${adb_dnsfilereset}" -eq 1 ] - then - printf "${adb_dnsheader}" > "${adb_dnsdir}/${adb_dnsfile}" - f_log "info" "blocklist with overall ${adb_cnt} blocked domains loaded successfully and reset afterwards (${adb_sysver})" - f_bgserv "start" - else - f_log "info" "blocklist with overall ${adb_cnt} blocked domains loaded successfully (${adb_sysver})" - fi + f_log "info" "blocklist with overall ${adb_cnt} blocked domains loaded successfully (${adb_sysver})" else f_log "err" "dns backend restart with adblock blocklist failed" fi @@ -1419,10 +1425,10 @@ f_report() { local iface bg_pid status total start end blocked percent top_list top array item index hold ports cnt=0 search="${1}" count="${2}" process="${3}" print="${4}" - if [ "${adb_report}" -eq 1 ] && [ ! -x "${adb_dumpcmd}" ] + if [ "${adb_report}" = "1" ] && [ ! -x "${adb_dumpcmd}" ] then f_log "info" "Please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" - elif [ "${adb_report}" -eq 0 ] && [ "${adb_action}" = "report" ] + elif [ "${adb_report}" = "0" ] && [ "${adb_action}" = "report" ] then f_log "info" "Please enable the 'DNS Report' option to use the reporting feature" fi @@ -1430,7 +1436,7 @@ f_report() if [ -x "${adb_dumpcmd}" ] then bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - if [ "${adb_report}" -eq 0 ] || { [ -n "${bg_pid}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; } } + if [ "${adb_report}" = "0" ] || { [ -n "${bg_pid}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; } } then if [ -n "${bg_pid}" ] then @@ -1444,7 +1450,7 @@ f_report() fi fi - if [ -x "${adb_dumpcmd}" ] && [ "${adb_report}" -eq 1 ] + if [ -x "${adb_dumpcmd}" ] && [ "${adb_report}" = "1" ] then if [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ] then @@ -1501,7 +1507,7 @@ f_report() e=$(NF-1);sub(/[0-9]\/[0-9]\/[0-9]|0\.0\.0\.0/,"NX",e);sub(/\.$/,"",e);sub(/([0-9]{1,3}\.){3}[0-9]{1,3}/,"OK",e);gsub(/[^[:alnum:]\.-]/,"",e);if(e==""){e="err"};printf "%s\t%s\t%s\t%s\t%s\n",d,e,a,b,c}' >> "${adb_reportdir}/adb_report.raw" )& hold=$((cnt%adb_maxqueue)) - if [ "${hold}" -eq 0 ] + if [ "${hold}" = "0" ] then wait fi @@ -1524,38 +1530,38 @@ f_report() blocked="$("${adb_awk}" '{if($5=="NX")cnt++}END{printf "%s",cnt}' "${adb_reportdir}/adb_report.srt")" percent="$("${adb_awk}" -v t="${total}" -v b="${blocked}" 'BEGIN{printf "%.2f%s",b/t*100,"%"}')" > "${adb_reportdir}/adb_report.json" - printf "%s" "{ \"data\": { " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"start_date\": \"${start%_*}\", " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"start_time\": \"${start#*_}\", " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"end_date\": \"${end%_*}\", " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"end_time\": \"${end#*_}\", " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"total\": \"${total}\", " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"blocked\": \"${blocked}\", " >> "${adb_reportdir}/adb_report.json" - printf "%s" "\"percent\": \"${percent}\", " >> "${adb_reportdir}/adb_report.json" + printf "%s\n" "{ " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"start_date\": \"${start%_*}\", " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"start_time\": \"${start#*_}\", " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"end_date\": \"${end%_*}\", " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"end_time\": \"${end#*_}\", " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"total\": \"${total}\", " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"blocked\": \"${blocked}\", " >> "${adb_reportdir}/adb_report.json" + printf "\t%s\n" "\"percent\": \"${percent}\", " >> "${adb_reportdir}/adb_report.json" top_list="top_clients top_domains top_blocked" for top in ${top_list} do - printf "%s" " \"${top}\": [ " >> "${adb_reportdir}/adb_report.json" + printf "\t%s" "\"${top}\": [ " >> "${adb_reportdir}/adb_report.json" case "${top}" in "top_clients") "${adb_awk}" '{print $3}' "${adb_reportdir}/adb_report.srt" | "${adb_sort}" ${adb_srtopts} | uniq -c | \ - "${adb_sort}" ${adb_srtopts} -nr | "${adb_awk}" '{ORS=" ";if(NR==1)printf "{ \"count\": \"%s\", \"address\": \"%s\" }",$1,$2; else if(NR<10)printf ", { \"count\": \"%s\", \"address\": \"%s\" }",$1,$2}' >> "${adb_reportdir}/adb_report.json" + "${adb_sort}" ${adb_srtopts} -nr | "${adb_awk}" '{ORS=" ";if(NR==1)printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}",$1,$2; else if(NR<10)printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}",$1,$2}' >> "${adb_reportdir}/adb_report.json" ;; "top_domains") "${adb_awk}" '{if($5!="NX")print $4}' "${adb_reportdir}/adb_report.srt" | "${adb_sort}" ${adb_srtopts} | uniq -c | \ - "${adb_sort}" ${adb_srtopts} -nr | "${adb_awk}" '{ORS=" ";if(NR==1)printf "{ \"count\": \"%s\", \"address\": \"%s\" }",$1,$2; else if(NR<10)printf ", { \"count\": \"%s\", \"address\": \"%s\" }",$1,$2}' >> "${adb_reportdir}/adb_report.json" + "${adb_sort}" ${adb_srtopts} -nr | "${adb_awk}" '{ORS=" ";if(NR==1)printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}",$1,$2; else if(NR<10)printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}",$1,$2}' >> "${adb_reportdir}/adb_report.json" ;; "top_blocked") "${adb_awk}" '{if($5=="NX")print $4}' "${adb_reportdir}/adb_report.srt" | "${adb_sort}" ${adb_srtopts} | uniq -c | \ - "${adb_sort}" ${adb_srtopts} -nr | "${adb_awk}" '{ORS=" ";if(NR==1)printf "{ \"count\": \"%s\", \"address\": \"%s\" }",$1,$2; else if(NR<10)printf ", { \"count\": \"%s\", \"address\": \"%s\" }",$1,$2}' >> "${adb_reportdir}/adb_report.json" + "${adb_sort}" ${adb_srtopts} -nr | "${adb_awk}" '{ORS=" ";if(NR==1)printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}",$1,$2; else if(NR<10)printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}",$1,$2}' >> "${adb_reportdir}/adb_report.json" ;; esac - printf "%s" " ], " >> "${adb_reportdir}/adb_report.json" + printf "\n\t%s\n" "]," >> "${adb_reportdir}/adb_report.json" done search="${search//./\\.}" search="${search//[+*~%\$&\"\' ]/}" - "${adb_awk}" "BEGIN{i=0;printf \"\\\"requests\\\": [ \" }/(${search})/{i++;if(i==1)printf \"{ \\\"date\\\": \\\"%s\\\", \\\"time\\\": \\\"%s\\\", \\\"client\\\": \\\"%s\\\", \\\"domain\\\": \\\"%s\\\", \\\"rc\\\": \\\"%s\\\" }\",\$1,\$2,\$3,\$4,\$5;else if(i<=${count})printf \", { \\\"date\\\": \\\"%s\\\", \\\"time\\\": \\\"%s\\\", \\\"client\\\": \\\"%s\\\", \\\"domain\\\": \\\"%s\\\", \\\"rc\\\": \\\"%s\\\" }\",\$1,\$2,\$3,\$4,\$5}END{printf \" ] } }\n\"}" "${adb_reportdir}/adb_report.srt" >> "${adb_reportdir}/adb_report.json" + "${adb_awk}" "BEGIN{i=0;printf \"\t\\\"requests\\\": [\n\"}/(${search})/{i++;if(i==1)printf \"\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5;else if(i<=${count})printf \",\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5}END{printf \"\n\t]\n}\n\"}" "${adb_reportdir}/adb_report.srt" >> "${adb_reportdir}/adb_report.json" rm -f "${adb_reportdir}/adb_report.srt" fi fi @@ -1564,17 +1570,16 @@ f_report() then if [ "${print}" = "cli" ] then - printf "%s\\n%s\\n%s\\n" ":::" "::: Adblock DNS-Query Report" ":::" + printf "%s\n%s\n%s\n" ":::" "::: Adblock DNS-Query Report" ":::" json_load_file "${adb_reportdir}/adb_report.json" - json_select "data" json_get_keys keylist for key in ${keylist} do json_get_var value "${key}" eval "${key}=\"${value}\"" done - printf " + %s\\n + %s\\n" "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" - printf " + %s\\n + %s %s\\n" "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" + printf " + %s\n + %s\n" "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" + printf " + %s\n + %s %s\n" "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" top_list="top_clients top_domains top_blocked requests" for top in ${top_list} @@ -1592,25 +1597,25 @@ f_report() esac if json_get_type status "${top}" && [ "${top}" != "requests" ] && [ "${status}" = "array" ] then - printf "%s\\n%s\\n%s\\n" ":::" "${item}" ":::" + printf "%s\n%s\n%s\n" ":::" "${item}" ":::" json_select "${top}" index=1 while json_get_type status "${index}" && [ "${status}" = "object" ] do json_get_values item "${index}" - printf " + %-9s::: %s\\n" ${item} + printf " + %-9s::: %s\n" ${item} index=$((index+1)) done elif json_get_type status "${top}" && [ "${top}" = "requests" ] && [ "${status}" = "array" ] then - printf "%s\\n%s\\n%s\\n" ":::" "::: Latest DNS Queries" ":::" - printf "%-15s%-15s%-45s%-80s%s\\n" "Date" "Time" "Client" "Domain" "Answer" + printf "%s\n%s\n%s\n" ":::" "::: Latest DNS Queries" ":::" + printf "%-15s%-15s%-45s%-80s%s\n" "Date" "Time" "Client" "Domain" "Answer" json_select "${top}" index=1 while json_get_type status "${index}" && [ "${status}" = "object" ] do json_get_values item "${index}" - printf "%-15s%-15s%-45s%-80s%s\\n" ${item} + printf "%-15s%-15s%-45s%-80s%s\n" ${item} index=$((index+1)) done fi @@ -1650,7 +1655,7 @@ fi # sort selection # -adb_sort="$(command -v gnu-sort)" +adb_sort="$(command -v /usr/libexec/sort-coreutils)" if [ -z "${adb_sort}" ] then adb_sort="$(command -v sort)" @@ -1673,12 +1678,10 @@ fi f_load case "${adb_action}" in "stop") - f_bgserv "stop" f_report "+" "50" "false" "false" f_rmdns ;; "restart") - f_bgserv "stop" f_report "+" "50" "false" "false" f_rmdns f_env @@ -1703,7 +1706,6 @@ case "${adb_action}" in f_query "${2}" ;; "start"|"reload") - f_bgserv "stop" f_report "+" "50" "false" "false" f_env f_main diff --git a/net/adblock/files/adblock.sources b/net/adblock/files/adblock.sources index 191fa9179..7cc7a8561 100644 --- a/net/adblock/files/adblock.sources +++ b/net/adblock/files/adblock.sources @@ -7,7 +7,7 @@ "descurl": "https://github.com/AdAway/adaway.github.io" }, "adguard": { - "url": "https://filters.adtidy.org/windows/filters/15.txt", + "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", "rule": "BEGIN{FS=\"[\/|^|\\r]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+[\\/\\^\\r]+$/{print tolower($3)}", "size": "L", "focus": "general", @@ -55,38 +55,10 @@ "focus": "general", "descurl": "https://disconnect.me" }, - "energized_blugo": { - "url": "https://block.energized.pro/bluGo/formats/domains.txt", + "energized": { + "url": "https://block.energized.pro/", "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://energized.pro" - }, - "energized_blu": { - "url": "https://block.energized.pro/blu/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://energized.pro" - }, - "energized_porn": { - "url": "https://block.energized.pro/porn/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XXL", - "focus": "compilation+porn", - "descurl": "https://energized.pro" - }, - "energized_ultimate": { - "url": "https://block.energized.pro/ultimate/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XXL", - "focus": "compilation", - "descurl": "https://energized.pro" - }, - "energized_unified": { - "url": "https://block.energized.pro/unified/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XXL", + "size": "VAR", "focus": "compilation", "descurl": "https://energized.pro" }, @@ -111,13 +83,6 @@ "focus": "gaming", "descurl": "https://github.com/abyssin/pihole-blocklist" }, - "malwaredomains": { - "url": "https://mirror.cedia.org.ec/malwaredomains/justdomains", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "M", - "focus": "malware", - "descurl": "https://www.malwaredomains.com" - }, "notracking": { "url": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/dnscrypt-proxy/dnscrypt-proxy.blacklist.txt", "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", @@ -125,9 +90,16 @@ "focus": "tracking", "descurl": "https://github.com/notracking/hosts-blocklists" }, - "oisd_nl": { - "url": "https://abp.oisd.nl/", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", + "oisd_basic": { + "url": "https://dbl.oisd.nl/basic", + "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", + "size": "L", + "focus": "general", + "descurl": "https://oisd.nl" + }, + "oisd_full": { + "url": "https://dbl.oisd.nl", + "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", "size": "XXL", "focus": "general", "descurl": "https://oisd.nl" @@ -217,11 +189,11 @@ "descurl": "https://easylist.to" }, "reg_pl1": { - "url": "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts_without_controversies.txt", + "url": "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt", "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", "size": "S", "focus": "reg_poland", - "descurl": "https://kadantiscam.netlify.com" + "descurl": "https://kadantiscam.netlify.app" }, "reg_pl2": { "url": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-pihole-filters/hostfile.txt", @@ -237,6 +209,13 @@ "focus": "reg_romania", "descurl": "https://easylist.to" }, + "reg_se": { + "url": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt", + "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", + "size": "s", + "focus": "reg_sweden", + "descurl": "https://github.com/lassekongo83/Frellwits-filter-lists" + }, "reg_ru": { "url": "https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt", "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", @@ -254,22 +233,9 @@ "shallalist": { "url": "https://www.shallalist.de/Downloads/shallalist.tar.gz", "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "categories": [ - "adv", "costtraps", "spyware", "tracker", "warez" - ], - "size": "L", + "size": "VAR", "focus": "general", - "descurl": "https://www.shallalist.de" - }, - "shallalist_porn": { - "url": "https://www.shallalist.de/Downloads/shallalist.tar.gz", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "categories": [ - "adv", "costtraps", "porn", "spyware", "tracker", "warez" - ], - "size": "XXL", - "focus": "general+porn", - "descurl": "https://www.shallalist.de" + "descurl": "https://www.shallalist.de", }, "smarttv_tracking": { "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", @@ -286,16 +252,9 @@ "descurl": "https://github.com/Dawsey21" }, "stevenblack": { - "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/StevenBlack/hosts" - }, - "stevenblack_porn": { - "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn-social/hosts", + "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "L", + "size": "VAR", "focus": "compilation", "descurl": "https://github.com/StevenBlack/hosts" }, @@ -309,22 +268,9 @@ "utcapitole": { "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "categories": [ - "publicite", "bitcoin", "cryptojacking", "ddos", "malware", "phishing", "warez" - ], - "size": "L", + "size": "VAR", "focus": "general", - "descurl": "https://dsi.ut-capitole.fr/blacklists/index_en.php" - }, - "utcapitole_porn": { - "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "categories": [ - "adult", "publicite", "bitcoin", "cryptojacking", "ddos", "malware", "phishing", "warez" - ], - "size": "XXL", - "focus": "general+porn", - "descurl": "https://dsi.ut-capitole.fr/blacklists/index_en.php" + "descurl": "https://dsi.ut-capitole.fr/blacklists/index_en.php", }, "wally3k": { "url": "https://v.firebog.net/hosts/static/w3kbl.txt",