diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 28a7feeff..00abdf3a2 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=3.1.1 +PKG_VERSION:=3.4.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -17,6 +17,7 @@ define Package/adblock SECTION:=net CATEGORY:=Network TITLE:=Powerful adblock script to block ad/abuse domains + DEPENDS:=+jshn +jsonfilter PKGARCH:=all endef @@ -53,6 +54,7 @@ define Package/adblock/install $(INSTALL_CONF) ./files/adblock.conf $(1)/etc/config/adblock $(INSTALL_DIR) $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.notify $(1)/etc/adblock/ $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock/ $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock/ endef diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 6b373da52..7010c3eaf 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -43,8 +43,6 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => regional blocklist for Romania, weekly updates, approx. 600 entries * [reg_ru](https://code.google.com/p/ruadlist) * => regional blocklist for Russia, weekly updates, approx. 2.000 entries - * [securemecca](http://www.securemecca.com) - * => infrequent updates, approx. 25.000 entries * [shallalist](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default) * => daily updates, approx. 32.000 entries (a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html)) * [spam404](http://www.spam404.com) @@ -76,6 +74,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * minimal status & error logging to syslog, enable debug logging to receive more output * procd based init system support (start/stop/restart/reload/suspend/resume/query/status) * procd network interface trigger support or classic time based startup +* keep the dns cache intact after adblock processing (currently supported by unbound and named) * conditional dns backend restarts by old/new blocklist comparison with sha256sum (default) or md5sum * suspend & resume adblock actions temporarily without blocklist reloading * output comprehensive runtime information via LuCI or via 'status' init command @@ -86,6 +85,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * optional: automatic blocklist backup & restore, they will be used in case of download errors or during startup in backup mode * optional: 'backup mode' to re-use blocklist backups during startup, get fresh lists only via reload or restart action * optional: 'whitelist mode' to block access to all domains except those explicitly listed in the whitelist file +* optional: send notification emails in case of a processing error or if the overall domain count is ≤ 0 * optional: add new adblock sources on your own via uci config ## Prerequisites @@ -133,18 +133,21 @@ A lot of people already use adblocker plugins within their desktop browsers, but * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (default: '0', disabled) * adb\_backup_mode => do not automatically update blocklists during startup, use backups instead (default: '0', disabled) * adb\_whitelist_mode => block access to all domains except those explicitly listed in the whitelist file (default: '0', disabled) + * adb\_dnsflush => flush DNS cache after adblock processing, i.e. enable the old restart behavior (default: '0', disabled) + * adb\_notify => send notification emails in case of a processing error or if the overall domain count is ≤ 0 (default: '0', disabled) + * adb\_notifycnt => Raise minimum domain count email notification trigger (default: '0') ## Examples -**change default dns backend to 'unbound':** +**change default dns backend to 'unbound':** -Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' where unbound can find them in its jail. If you use manual configuration for unbound, then just include the following line in your 'server' clause: -

-  include: "/var/lib/unbound/adb_list.overall"
-
+Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' where unbound can find them in its jail. +To preserve the DNS cache after adblock processing you need to install 'unbound-control'. -**change default dns backend to 'named' (bind):** +**change default dns backend to 'named' (bind):** -Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind'. To use the blocklist please modify '/etc/bind/named.conf': +Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind'. +To preserve the DNS cache after adblock processing you need to install & configure 'bind-rdnc'. +To use the blocklist please modify '/etc/bind/named.conf':

 in the 'options' namespace add:
   response-policy { zone "rpz"; };
@@ -158,21 +161,45 @@ and at the end of the file add:
   };
 
-**change default dns backend to 'kresd':** +**change default dns backend to 'kresd':** + +The knot-resolver (kresd) is only available on Turris Omnia devices. +Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd', no further configuration needed. + +**change default dns backend to 'dnscrypt-proxy':** -The knot-resolver (kresd) is only available on turris omnia devices. Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd'. To use the blocklist please modify '/etc/config/resolver': +The required 'blacklist' option of dnscrypt-proxy is not enabled by default, because the package will be compiled without plugins support. +Take a custom LEDE build with plugins support to use this feature. Adblock deposits the final blocklist 'adb_list.overall' in '/tmp'. +To use the blocklist please modify '/etc/config/dnscrypt-proxy' per instance:

-    list rpz_file '/etc/kresd/adb_list.overall'
+  list blacklist 'domains:/tmp/adb_list.overall'
 
-**change default dns backend to 'dnscrypt-proxy':** +**enable email notification via msmtp:** -The required 'blacklist' option of dnscrypt-proxy is not enabled by default, because the package will be compiled without plugins support. Take a custom LEDE build with plugins support to use this feature. Adblock deposits the final blocklist 'adb_list.overall' in '/tmp'. To use the blocklist please modify '/etc/config/dnscrypt-proxy' per instance: +To use the email notification you have to install & configure the package 'msmtp'. +Modify the file '/etc/msmtprc':

-  list blacklist 'domains:/tmp/adb_list.overall'
+[...]
+defaults
+auth            on
+tls             on
+tls_certcheck   off
+timeout         5
+syslog          LOG_MAIL
+[...]
+account         adb_notify
+host            smtp.gmail.com
+port            587
+from            dev.adblock@gmail.com
+user            dev.adblock
+password        xxx
 
+Edit the file '/etc/adblock/adblock.notify' and change at least the 'mail_receiver'. +Finally make this file executable via 'chmod' and test it directly. If no more errors come up you can comment 'mail_debug', too. **configuration for different download utilities:** +

 wget (default):
   option adb_fetch '/usr/bin/wget'
@@ -192,24 +219,27 @@ curl:
 
**receive adblock runtime information:** +

 /etc/init.d/adblock status
 ::: adblock runtime information
   + adblock_status  : enabled
-  + adblock_version : 3.1.0
-  + overall_domains : 5117
+  + adblock_version : 3.4.0
+  + overall_domains : 5167 (normal/backup mode)
   + fetch_utility   : wget (built-in)
   + dns_backend     : kresd (/etc/kresd)
-  + last_rundate    : 03.11.2017 22:57:41
-  + system_release  : Turris Omnia, OpenWrt omnia 15.05/3.8.4
+  + last_rundate    : 27.12.2017 20:52:35
+  + system_release  : Turris Omnia, OpenWrt omnia 15.05/3.9.1
 
**cronjob for a regular block list update (/etc/crontabs/root):** +

 0 06 * * *    /etc/init.d/adblock reload
 
**blacklist entry (/etc/adblock/adblock.blacklist):** +

 ads.example.com
 
@@ -224,6 +254,7 @@ This entry does not block:
 
**whitelist entry (/etc/adblock/adblock.whitelist):** +

 here.com
 
@@ -236,7 +267,7 @@ This entry does not remove:
   www.adwhere.com
 
-**query the active blocklist for a certain (sub-)domain, e.g. for whitelisting:** +**query the active blocklist for a certain (sub-)domain, e.g. for whitelisting:** The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain. For every (sub-)domain it returns the first ten relevant results.

@@ -255,7 +286,7 @@ The query function checks against the submitted (sub-)domain and recurses automa
   + www-google-analytics.l.google.com
 
-**add a new blocklist source:** +**add a new blocklist source:** 1. the easy way ... example: https://easylist-downloads.adblockplus.org/rolist+easylist.txt diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index da2f6fe3b..960d7d923 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -126,12 +126,6 @@ config source 'reg_ru' option adb_src_rset 'BEGIN{FS=\"[|^]\"}\$0~/^\|\|([[:alnum:]_-]+\.){1,}[[:alpha:]]+\^("\\\$third-party")?$/{print tolower(\$3)}' option adb_src_desc 'focus on russian ads plus generic easylist additions, weekly updates, approx. 14.500 entries' -config source 'securemecca' - option enabled '0' - option adb_src 'http://securemecca.com/Downloads/hosts.txt' - option adb_src_rset '\$0~/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.){1,}[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}' - option adb_src_desc 'broad blocklist, infrequent updates, approx. 13.700 entries' - config source 'shalla' option enabled '0' option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz' diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index d49c6ea2d..b7b25f154 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -82,7 +82,7 @@ service_triggers() if [ "${trigger}" != "none" ] && [ "${trigger}" != "timed" ] then - PROCD_RELOAD_DELAY=$((${delay:=1} * 1000)) + PROCD_RELOAD_DELAY=$((${delay:-1} * 1000)) procd_add_interface_trigger "interface.*.up" "${trigger}" "${adb_init}" start fi procd_add_reload_trigger "adblock" diff --git a/net/adblock/files/adblock.notify b/net/adblock/files/adblock.notify new file mode 100644 index 000000000..3f8fc9723 --- /dev/null +++ b/net/adblock/files/adblock.notify @@ -0,0 +1,38 @@ +#!/bin/sh +# +# adblock send mail script for mstmp +# written by Dirk Brenken (dev@brenken.org) +# Please note: you have to install and configure the package 'mstmp' before using this script. + +# This is free software, licensed under the GNU General Public License v3. +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +LC_ALL=C +PATH="/usr/sbin:/usr/bin:/sbin:/bin" +mail_daemon="$(command -v sendmail)" +mail_profile="adb_notify" +mail_debug="--debug" +mail_rc=0 + +# mail header +# +mail_sender="no-reply@adblock" +mail_receiver="!!!ChangeMe!!!" +mail_topic="adblock notification" +mail_head="From: ${mail_sender}\nTo: ${mail_receiver}\nSubject: ${mail_topic}\nReply-to: ${mail_sender}\n\n" + +# mail body +# +mail_text="adblock status, generated at $(date 2>&1)!" +mail_text="${mail_text}\n++\n++ System Information ++\n++\n$(cat /etc/banner 2>&1)" +mail_text="${mail_text}\n\n++\n++ Adblock Information ++\n++\n$(/etc/init.d/adblock status 2>&1)" + +# send mail +# +if [ -x "${mail_daemon}" ] +then + printf "%b" "${mail_head}${mail_text}" | "${mail_daemon}" ${mail_debug} -a "${mail_profile}" "${mail_receiver}" 2>&1 + mail_rc=${?} +fi +exit ${mail_rc} diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index 209fc00fe..7d2a35fe5 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,7 +10,7 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="3.1.1" +adb_ver="3.4.0" adb_sysver="unknown" adb_enabled=0 adb_debug=0 @@ -18,22 +18,27 @@ adb_backup_mode=0 adb_whitelist_mode=0 adb_forcesrt=0 adb_forcedns=0 +adb_notify=0 +adb_notifycnt=0 adb_triggerdelay=0 adb_backup=0 adb_backupdir="/mnt" -adb_fetch="/usr/bin/wget" +adb_fetchutil="/usr/bin/wget" adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O" adb_dns="dnsmasq" adb_dnsprefix="adb_list" adb_dnsfile="${adb_dnsprefix}.overall" +adb_dnsflush=0 adb_whitelist="/etc/adblock/adblock.whitelist" adb_rtfile="/tmp/adb_runtime.json" -adb_hashsum="$(command -v sha256sum)" -adb_action="${1:-"start"}" +adb_hashutil="$(command -v sha256sum)" +adb_hashold="" +adb_hashnew="" adb_cnt=0 adb_rc=0 +adb_action="${1:-"start"}" -# f_envload: load adblock environment +# load adblock environment # f_envload() { @@ -54,6 +59,13 @@ f_envload() adb_sysver="${sys_model}, ${sys_desc}" fi + # check hash utility + # + if [ ! -x "${adb_hashutil}" ] + then + adb_hashutil="$(command -v md5sum)" + fi + # source in system libraries # if [ -r "/lib/functions.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ] @@ -103,30 +115,35 @@ f_envload() config_load adblock config_foreach parse_config source - # set/check dns backend environment + # check dns backend # case "${adb_dns}" in dnsmasq) + adb_dnsinstance=0 adb_dnsuser="${adb_dnsuser:-"dnsmasq"}" - adb_dnsdir="${adb_dnsdir:-"/tmp/dnsmasq.d"}" - adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'" + adb_dnsdir="${adb_dnsdir:-"/tmp"}" + adb_dnsheader="" + adb_dnsformat="awk '{print \"server=/\"\$0\"/\"}'" if [ ${adb_whitelist_mode} -eq 1 ] then - adb_dnsformat="awk '{print \"local=/\"\$0\"/#\"}'" - adb_dnsblock="local=/#/" + adb_dnsformat="awk '{print \"server=/\"\$0\"/#\"}'" + adb_dnsblock="server=/#/" fi - ;; + ;; unbound) + adb_dnsinstance=0 adb_dnsuser="${adb_dnsuser:-"unbound"}" adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" + adb_dnsheader="" adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 static\"}'" if [ ${adb_whitelist_mode} -eq 1 ] then adb_dnsformat="awk '{print \"local-zone: \042\"\$0\"\042 transparent\"}'" adb_dnsblock="local-zone: \".\" static" fi - ;; + ;; named) + adb_dnsinstance=0 adb_dnsuser="${adb_dnsuser:-"bind"}" adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)"$'\n'" IN NS localhost." @@ -136,8 +153,9 @@ f_envload() adb_dnsformat="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" adb_dnsblock="* CNAME ." fi - ;; + ;; kresd) + adb_dnsinstance=0 adb_dnsuser="${adb_dnsuser:-"root"}" adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" adb_dnsheader="\$TTL 2h"$'\n'"@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)"$'\n'" IN NS localhost." @@ -147,24 +165,22 @@ f_envload() adb_dnsformat="awk '{print \"\"\$0\" CNAME rpz-passthru.\n*.\"\$0\" CNAME rpz-passthru.\"}'" adb_dnsblock="* CNAME ." fi - ;; + ;; dnscrypt-proxy) + adb_dnsinstance=0 adb_dnsuser="${adb_dnsuser:-"nobody"}" adb_dnsdir="${adb_dnsdir:-"/tmp"}" + adb_dnsheader="" adb_dnsformat="awk '{print \$0}'" - ;; + ;; esac # check adblock status # if [ ${adb_enabled} -eq 0 ] then - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] - then - f_rmdns - f_dnsrestart - fi f_extconf + f_rmdns f_jsnupdate f_log "info " "adblock is currently disabled, please set adb_enabled to '1' to use this service" exit 0 @@ -172,7 +188,7 @@ f_envload() if [ -d "${adb_dnsdir}" ] && [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] then - > "${adb_dnsdir}/${adb_dnsfile}" + printf '%s\n' "${adb_dnsheader}" > "${adb_dnsdir}/${adb_dnsfile}" fi case "${adb_action}" in @@ -202,12 +218,14 @@ f_envload() fi } -# f_envcheck: check/set environment prerequisites +# check environment # f_envcheck() { local ssl_lib + f_log "info " "start adblock processing (${adb_action})" + # check external uci config files # f_extconf @@ -215,23 +233,23 @@ f_envcheck() # check fetch utility # ssl_lib="-" - if [ -x "${adb_fetch}" ] + if [ -x "${adb_fetchutil}" ] then - if [ "$(readlink -fn "${adb_fetch}")" = "/usr/bin/wget-nossl" ] + if [ "$(readlink -fn "${adb_fetchutil}")" = "/usr/bin/wget-nossl" ] then adb_fetchparm="--quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 -O" - elif [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] || - ([ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] && [ "$(readlink -fn "${adb_fetch}")" != "/usr/bin/wget" ]) + elif [ "$(readlink -fn "${adb_fetchutil}")" = "/bin/busybox" ] || + ([ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] && [ "$(readlink -fn "${adb_fetchutil}")" != "/usr/bin/wget" ]) then - adb_fetch="/bin/busybox" + adb_fetchutil="/bin/busybox" adb_fetchparm="-q -O" else ssl_lib="built-in" fi fi - if [ ! -x "${adb_fetch}" ] && [ "$(readlink -fn "/bin/wget")" = "/bin/uclient-fetch" ] + if [ ! -x "${adb_fetchutil}" ] && [ "$(readlink -fn "/bin/wget")" = "/bin/uclient-fetch" ] then - adb_fetch="/bin/uclient-fetch" + adb_fetchutil="/bin/uclient-fetch" if [ -f "/lib/libustream-ssl.so" ] then adb_fetchparm="-q --timeout=10 --no-check-certificate -O" @@ -240,18 +258,11 @@ f_envcheck() adb_fetchparm="-q --timeout=10 -O" fi fi - if [ ! -x "${adb_fetch}" ] || [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ] + if [ ! -x "${adb_fetchutil}" ] || [ -z "${adb_fetchutil}" ] || [ -z "${adb_fetchparm}" ] then f_log "error" "no download utility found, please install 'uclient-fetch' with 'libustream-mbedtls' or the full 'wget' package" fi - adb_fetchinfo="${adb_fetch##*/} (${ssl_lib})" - - # check hashsum utility - # - if [ ! -x "${adb_hashsum}" ] - then - adb_hashsum="$(command -v md5sum)" - fi + adb_fetchinfo="${adb_fetchutil##*/} (${ssl_lib})" # initialize temp files and directories # @@ -261,30 +272,119 @@ f_envcheck() > "${adb_tmpdir}/tmp.whitelist" } -# f_extconf: set external config options +# remove temporary files and directories # -f_extconf() +f_rmtemp() { - # kresd related options - # - if [ "${adb_dns}" = "kresd" ] + if [ -d "${adb_tmpdir}" ] then - if [ ${adb_enabled} -eq 1 ] && [ -z "$(uci -q get resolver.kresd.rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" ] - then - uci -q add_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - elif [ ${adb_enabled} -eq 0 ] && [ -n "$(uci -q get resolver.kresd.rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" ] + rm -f "${adb_tmpload}" + rm -f "${adb_tmpfile}" + rm -rf "${adb_tmpdir}" + fi +} + +# remove dns related files and directories +# +f_rmdns() +{ + if [ -n "${adb_dns}" ] + then + f_hashcompare + printf '%s\n' "${adb_dnsheader}" > "${adb_dnsdir}/${adb_dnsfile}" + > "${adb_dnsdir}/.${adb_dnsfile}" + > "${adb_rtfile}" + rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz + f_hashcompare + if [ ${?} -eq 1 ] then - uci -q del_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" + f_dnsrestart fi - if [ -n "$(uci -q changes resolver)" ] + fi +} + +# commit uci changes +# +f_ucichange() +{ + local config="${1}" + + if [ -n "$(uci -q changes "${config}")" ] + then + uci -q commit "${config}" + case "${config}" in + dhcp) + /etc/init.d/"${adb_dns}" reload >/dev/null 2>&1 + ;; + firewall) + /etc/init.d/firewall reload >/dev/null 2>&1 + ;; + esac + fi +} + +# list/overall count +# +f_count() +{ + local mode="${1}" + + adb_cnt=0 + if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && ([ -z "${mode}" ] || [ "${mode}" = "format" ]) + then + if [ "${adb_dns}" = "named" ] || [ "${adb_dns}" = "kresd" ] then - uci -q commit resolver + adb_cnt="$(( ( $(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^") ) / 2 ))" + else + adb_cnt="$(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}")" fi + elif [ -s "${adb_tmpfile}" ] + then + adb_cnt="$(wc -l 2>/dev/null < "${adb_tmpfile}")" fi +} - # firewall related options - # - if [ ${adb_enabled} -eq 1 ] && [ ${adb_forcedns} -eq 1 ] && [ -z "$(uci -q get firewall.adblock_dns)" ] +# set external config options +# +f_extconf() +{ + local uci_config + + case "${adb_dns}" in + dnsmasq) + uci_config="dhcp" + if [ ${adb_enabled} -eq 1 ] && [ -z "$(uci -q get dhcp.@dnsmasq[${adb_dnsinstance}].serversfile | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" ] + then + uci -q set dhcp.@dnsmasq[${adb_dnsinstance}].serversfile="${adb_dnsdir}/${adb_dnsfile}" + elif [ ${adb_enabled} -eq 0 ] && [ -n "$(uci -q get dhcp.@dnsmasq[${adb_dnsinstance}].serversfile | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" ] + then + uci -q delete dhcp.@dnsmasq[${adb_dnsinstance}].serversfile + fi + ;; + kresd) + uci_config="resolver" + if [ ${adb_enabled} -eq 1 ] && [ -z "$(uci -q get resolver.kresd.rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" ] + then + uci -q add_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" + elif [ ${adb_enabled} -eq 0 ] && [ -n "$(uci -q get resolver.kresd.rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" ] + then + uci -q del_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" + fi +# wait for upstream change +# if [ ${adb_enabled} -eq 1 ] && [ ${adb_dnsflush} -eq 0 ] && [ "$(uci -q get resolver.common.msg_keep_cache)" != "1" ] +# then +# uci -q set resolver.common.msg_keep_cache="1" +# elif [ ${adb_enabled} -eq 0 ] || ([ ${adb_dnsflush} -eq 1 ] && [ "$(uci -q get resolver.common.msg_keep_cache)" = "1" ]) +# then +# uci -q delete resolver.common.msg_keep_cache +# fi + ;; + esac + f_ucichange "${uci_config}" + + uci_config="firewall" + if [ ${adb_enabled} -eq 1 ] && [ ${adb_forcedns} -eq 1 ] && \ + [ -z "$(uci -q get firewall.adblock_dns)" ] && [ $(/etc/init.d/firewall enabled; printf "%u" ${?}) -eq 0 ] then uci -q set firewall.adblock_dns="redirect" uci -q set firewall.adblock_dns.name="Adblock DNS" @@ -297,117 +397,146 @@ f_extconf() then uci -q delete firewall.adblock_dns fi - if [ -n "$(uci -q changes firewall)" ] - then - uci -q commit firewall - if [ $(/etc/init.d/firewall enabled; printf "%u" ${?}) -eq 0 ] - then - /etc/init.d/firewall reload >/dev/null 2>&1 - fi - fi + f_ucichange "${uci_config}" } -# f_rmtemp: remove temporary files & directories +# restart of the dns backend # -f_rmtemp() +f_dnsrestart() { - if [ -d "${adb_tmpdir}" ] - then - rm -f "${adb_tmpload}" - rm -f "${adb_tmpfile}" - rm -rf "${adb_tmpdir}" - fi -} + local dns_up cache_util cache_rc out_rc=1 cnt=0 -# f_rmdns: remove dns related files & directories -# -f_rmdns() -{ - if [ -n "${adb_dns}" ] + if [ ${adb_dnsflush} -eq 0 ] && [ ${adb_enabled} -eq 1 ] && \ + [ "${adb_rc}" -eq 0 ] && [ "${adb_action}" != "stop" ] then - > "${adb_dnsdir}/${adb_dnsfile}" - > "${adb_rtfile}" - rm -f "${adb_dnsdir}/.${adb_dnsfile}" - rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz + case "${adb_dns}" in + dnsmasq) + cache_util="n/a" + killall -q -HUP "${adb_dns}" + cache_rc=${?} + ;; + unbound) + cache_util="$(command -v unbound-control)" + if [ -x "${cache_util}" ] && [ -f "${adb_dnsdir}"/unbound.conf ] + then + "${cache_util}" -c "${adb_dnsdir}"/unbound.conf dump_cache > "${adb_tmpdir}"/adb_cache.dump 2>/dev/null + fi + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + ;; + kresd) + cache_util="n/a" + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + cache_rc=${?} + ;; + named) + cache_util="$(command -v rndc)" + if [ -x "${cache_util}" ] && [ -f /etc/bind/rndc.conf ] + then + "${cache_util}" -c /etc/bind/rndc.conf reload >/dev/null 2>&1 + cache_rc=${?} + else + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + fi + ;; + *) + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + ;; + esac + else + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 fi -} - -# f_dnsrestart: restart the dns backend -# -f_dnsrestart() -{ - local dns_up cnt=0 - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 while [ ${cnt} -le 10 ] do dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running")" if [ "${dns_up}" = "true" ] then - return 0 + case "${adb_dns}" in + unbound) + cache_util="$(command -v unbound-control)" + if [ -x "${cache_util}" ] && [ -s "${adb_tmpdir}"/adb_cache.dump ] + then + while [ ${cnt} -le 10 ] + do + "${cache_util}" -c "${adb_dnsdir}"/unbound.conf load_cache < "${adb_tmpdir}"/adb_cache.dump >/dev/null 2>&1 + cache_rc=${?} + if [ ${cache_rc} -eq 0 ] + then + break + fi + cnt=$((cnt+1)) + sleep 1 + done + fi + ;; + esac + out_rc=0 + break fi cnt=$((cnt+1)) sleep 1 done - return 1 + f_log "debug" "cache_util: ${cache_util}, cache_rc: ${cache_rc}, cache_flush: ${adb_dnsflush}, count: ${cnt}, out_rc: ${out_rc}" + return ${out_rc} } -# f_list: backup/restore/remove blocklists +# backup/restore/remove blocklists # f_list() { - local mode="${1}" in_rc="${adb_rc}" cnt=0 + local mode="${1}" in_rc="${adb_rc}" case "${mode}" in backup) - cnt="$(wc -l < "${adb_tmpfile}")" if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] then - gzip -cf "${adb_tmpfile}" > "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" + gzip -cf "${adb_tmpfile}" > "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null adb_rc=${?} fi - ;; + ;; restore) if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ] && [ -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ] then - gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}" + gunzip -cf "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}" 2>/dev/null adb_rc=${?} fi - ;; + ;; remove) if [ -d "${adb_backupdir}" ] then rm -f "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" fi adb_rc=${?} - ;; + ;; merge) if [ -s "${adb_tmpfile}" ] then cat "${adb_tmpfile}" >> "${adb_tmpdir}/${adb_dnsfile}" adb_rc=${?} fi - ;; + ;; format) + src_name="overall" if [ -s "${adb_tmpdir}/tmp.whitelist" ] then grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpdir}/${adb_dnsfile}" | eval "${adb_dnsformat}" > "${adb_dnsdir}/${adb_dnsfile}" else eval "${adb_dnsformat}" "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" fi - if [ -n "${adb_dnsheader}" ] + if [ ${?} -eq 0 ] && [ -n "${adb_dnsheader}" ] then printf '%s\n' "${adb_dnsheader}" | cat - "${adb_dnsdir}/${adb_dnsfile}" > "${adb_tmpdir}/${adb_dnsfile}" cat "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" fi adb_rc=${?} - ;; + ;; esac - f_log "debug" "name: ${src_name}, mode: ${mode}, count: ${cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" + f_count "${mode}" + f_log "debug" "name: ${src_name}, mode: ${mode}, count: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" } -# f_tldcompression: top level domain compression +# top level domain compression # f_tldcompression() { @@ -419,35 +548,64 @@ f_tldcompression() sort -u "${temp}" > "${source}" } -# f_switch: suspend/resume adblock processing +# blocklist hash compare +# +f_hashcompare() +{ + local hash out_rc=1 + + if [ -x "${adb_hashutil}" ] && [ -f "${adb_dnsdir}/${adb_dnsfile}" ] + then + hash="$(${adb_hashutil} "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null | awk '{print $1}')" + if [ -z "${adb_hashold}" ] && [ -n "${hash}" ] + then + adb_hashold="${hash}" + elif [ -z "${adb_hashnew}" ] && [ -n "${hash}" ] + then + adb_hashnew="${hash}" + fi + if [ -n "${adb_hashold}" ] && [ -n "${adb_hashnew}" ] + then + if [ "${adb_hashold}" = "${adb_hashnew}" ] + then + out_rc=0 + fi + adb_hashold="" + adb_hashnew="" + fi + fi + f_log "debug" "hash_util: ${adb_hashutil}, hash: ${hash}, out_rc: ${out_rc}" + return ${out_rc} +} + +# suspend/resume adblock processing # f_switch() { - local source target status mode="${1}" + local mode="${1}" - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] && [ "${mode}" = "suspend" ] + if [ ! -s "${adb_dnsdir}/.${adb_dnsfile}" ] && [ "${mode}" = "suspend" ] then - source="${adb_dnsdir}/${adb_dnsfile}" - target="${adb_dnsdir}/.${adb_dnsfile}" - status="suspended" + f_hashcompare + cat "${adb_dnsdir}/${adb_dnsfile}" > "${adb_dnsdir}/.${adb_dnsfile}" + printf '%s\n' "${adb_dnsheader}" > "${adb_dnsdir}/${adb_dnsfile}" + f_hashcompare elif [ -s "${adb_dnsdir}/.${adb_dnsfile}" ] && [ "${mode}" = "resume" ] then - source="${adb_dnsdir}/.${adb_dnsfile}" - target="${adb_dnsdir}/${adb_dnsfile}" - status="resumed" + f_hashcompare + cat "${adb_dnsdir}/.${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" + > "${adb_dnsdir}/.${adb_dnsfile}" + f_hashcompare fi - if [ -n "${status}" ] + if [ ${?} -eq 1 ] then - cat "${source}" > "${target}" - > "${source}" - chown "${adb_dnsuser}" "${target}" 2>/dev/null f_dnsrestart f_jsnupdate - f_log "info " "adblock processing ${status}" + f_log "info " "${mode} adblock processing" fi } -# f_query: query blocklist for certain (sub-)domains +# query blocklist for certain (sub-)domains # f_query() { @@ -455,10 +613,7 @@ f_query() local domain="${1}" local tld="${domain#*.}" - if [ ! -s "${adb_dnsdir}/${adb_dnsfile}" ] - then - printf "%s\n" "::: no active blocklist found, please start / resume adblock first" - elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] + if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then printf "%s\n" "::: invalid domain input, please submit a single domain, e.g. 'doubleclick.net'" else @@ -472,7 +627,7 @@ f_query() else result="$(awk "/(^[^\*][[:alpha:]]*[\.]+${search}|^${search})/{i++;{printf(\" + %s\n\",\$1)};if(i>9){exit}}" "${adb_dnsfile}")" fi - printf "%s\n" "::: max. ten results for domain '${domain}'" + printf "%s\n" "::: results for domain '${domain}' (max. 10)" printf "%s\n" "${result:-" - no match"}" domain="${tld}" tld="${domain#*.}" @@ -480,11 +635,11 @@ f_query() fi } -# f_jsnupdate: update runtime information +# update runtime information # f_jsnupdate() { - local status rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" + local status mode="normal mode" rundate="$(/bin/date "+%d.%m.%Y %H:%M:%S")" if [ ${adb_rc} -gt 0 ] then @@ -497,15 +652,15 @@ f_jsnupdate() status="paused" else status="enabled" - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] - then - if [ "${adb_dns}" = "named" ] || [ "${adb_dns}" = "kresd" ] - then - adb_cnt="$(( ( $(wc -l < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^") ) / 2 ))" - else - adb_cnt="$(wc -l < "${adb_dnsdir}/${adb_dnsfile}")" - fi - fi + f_count + fi + + if [ ${adb_backup_mode} -eq 1 ] + then + mode="normal/backup mode" + elif [ ${adb_whitelist_mode} -eq 1 ] + then + mode="whitelist mode" fi if [ -z "${adb_fetchinfo}" ] && [ -s "${adb_rtfile}" ] @@ -519,16 +674,22 @@ f_jsnupdate() json_add_object "data" json_add_string "adblock_status" "${status}" json_add_string "adblock_version" "${adb_ver}" - json_add_string "overall_domains" "${adb_cnt}" + json_add_string "overall_domains" "${adb_cnt} (${mode})" json_add_string "fetch_utility" "${adb_fetchinfo}" json_add_string "dns_backend" "${adb_dns} (${adb_dnsdir})" json_add_string "last_rundate" "${rundate}" json_add_string "system_release" "${adb_sysver}" json_close_object json_dump > "${adb_rtfile}" + + if [ ${adb_notify} -eq 1 ] && [ -x /etc/adblock/adblock.notify ] && ([ "${status}" = "error" ] || \ + ([ "${status}" = "enabled" ] && [ ${adb_whitelist_mode} -eq 0 ] && [ ${adb_cnt} -le ${adb_notifycnt} ])) + then + (/etc/adblock/adblock.notify >/dev/null 2>&1) & + fi } -# f_status: output runtime information +# output runtime information # f_status() { @@ -548,7 +709,7 @@ f_status() fi } -# f_log: write to syslog, exit on error +# write to syslog, exit on error # f_log() { @@ -559,15 +720,11 @@ f_log() logger -t "adblock-[${adb_ver}] ${class}" "${log_msg}" if [ "${class}" = "error" ] then - logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})" - f_rmtemp - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ] - then - f_rmdns - f_dnsrestart - fi adb_rc=1 + f_rmdns f_jsnupdate + f_rmtemp + logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})" exit 1 fi fi @@ -577,11 +734,9 @@ f_log() # f_main() { - local src_name src_rset shalla_archive enabled url hash_old hash_new - local mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" + local src_name src_rset shalla_archive enabled url mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" - f_log "info " "start adblock processing ..." - f_log "debug" "action: ${adb_action}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, hashsum: ${adb_hashsum}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, whitelist_mode: ${adb_whitelist_mode}, force_srt/_dns: ${adb_forcesrt}/${adb_forcedns}, mem_total: ${mem_total}" + f_log "debug" "dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, whitelist_mode: ${adb_whitelist_mode}, force_srt/_dns: ${adb_forcesrt}/${adb_forcedns}, mem_total: ${mem_total}" > "${adb_rtfile}" > "${adb_dnsdir}/.${adb_dnsfile}" @@ -598,10 +753,11 @@ f_main() awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist" fi - # whitelist mode + # whitelist mode processing # if [ ${adb_whitelist_mode} -eq 1 ] && [ "${adb_dns}" != "dnscrypt-proxy" ] then + f_hashcompare f_tldcompression "${adb_tmpdir}/tmp.whitelist" eval "${adb_dnsformat}" "${adb_tmpdir}/tmp.whitelist" > "${adb_dnsdir}/${adb_dnsfile}" printf '%s\n' "${adb_dnsblock}" >> "${adb_dnsdir}/${adb_dnsfile}" @@ -610,7 +766,11 @@ f_main() printf '%s\n' "${adb_dnsheader}" | cat - "${adb_dnsdir}/${adb_dnsfile}" > "${adb_tmpdir}/${adb_dnsfile}" cat "${adb_tmpdir}/${adb_dnsfile}" > "${adb_dnsdir}/${adb_dnsfile}" fi - f_dnsrestart + f_hashcompare + if [ ${?} -eq 1 ] + then + f_dnsrestart + fi if [ ${?} -eq 0 ] then f_jsnupdate "${adb_cnt}" @@ -618,10 +778,11 @@ f_main() else f_log "error" "dns backend restart with active whitelist failed" fi + f_rmtemp return fi - # normal & backup mode + # normal & backup mode processing # for src_name in ${adb_sources} do @@ -634,7 +795,7 @@ f_main() # basic pre-checks # - f_log "debug" "name: ${src_name}, enabled: ${enabled}, url: ${url}, rset: ${src_rset}" + f_log "debug" "name: ${src_name}, enabled: ${enabled}" if [ "${enabled}" != "1" ] || [ -z "${url}" ] || [ -z "${src_rset}" ] then f_list remove @@ -662,7 +823,7 @@ f_main() elif [ "${src_name}" = "shalla" ] then shalla_archive="${adb_tmpdir}/shallalist.tar.gz" - "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null + "${adb_fetchutil}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null adb_rc=${?} if [ ${adb_rc} -eq 0 ] then @@ -679,11 +840,11 @@ f_main() rm -f "${shalla_archive}" rm -rf "${adb_tmpdir}/BL" else - "${adb_fetch}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>/dev/null + "${adb_fetchutil}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>/dev/null adb_rc=${?} fi - # check download result and prepare list output (incl. tld compression, list backup & restore) + # check download result and prepare list output # if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] then @@ -716,12 +877,9 @@ f_main() fi done - # hash preparation, whitelist removal and overall sort + # overall sort and conditional dns restart # - if [ -x "${adb_hashsum}" ] && [ -f "${adb_dnsdir}/${adb_dnsfile}" ] - then - hash_old="$(${adb_hashsum} "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null | awk '{print $1}')" - fi + f_hashcompare if [ -s "${adb_tmpdir}/${adb_dnsfile}" ] then if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] @@ -733,15 +891,9 @@ f_main() > "${adb_dnsdir}/${adb_dnsfile}" fi chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null - f_rmtemp - # conditional restart of the dns backend and runtime information export - # - if [ -x "${adb_hashsum}" ] && [ -f "${adb_dnsdir}/${adb_dnsfile}" ] - then - hash_new="$(${adb_hashsum} "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null | awk '{print $1}')" - fi - if [ -z "${hash_old}" ] || [ -z "${hash_new}" ] || [ "${hash_old}" != "${hash_new}" ] + f_hashcompare + if [ ${?} -eq 1 ] then f_dnsrestart fi @@ -752,6 +904,7 @@ f_main() else f_log "error" "dns backend restart with active blocklist failed" fi + f_rmtemp } # handle different adblock actions @@ -759,31 +912,28 @@ f_main() f_envload case "${adb_action}" in stop) - f_rmtemp f_rmdns - f_dnsrestart - ;; + ;; restart) - f_rmtemp - f_rmdns f_envcheck + f_rmdns f_main - ;; + ;; suspend) f_switch suspend - ;; + ;; resume) f_switch resume - ;; + ;; query) f_query "${2}" - ;; + ;; status) f_status - ;; + ;; *) f_envcheck f_main - ;; + ;; esac exit 0