From a869954089b4bd3a65c65b6ca81daf0a78e4777a Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Sun, 28 Feb 2016 21:26:41 +0100 Subject: [PATCH] adblock: 0.90.0 * all relevant adblock events will be properly written to syslog/stdout * removed needless 'debug log' option * add optional parm 'adb_forcedns' to redirect all queries to local resolver (default: '1', enabled) * revised space check * various code cosmetics & cleanups Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 5 +- net/adblock/files/README.md | 19 +- net/adblock/files/adblock-helper.sh | 194 ++++++++------------- net/adblock/files/adblock-update.sh | 108 ++++-------- net/adblock/files/adblock.conf | 11 +- net/adblock/files/adblock.hotplug | 8 +- net/adblock/files/adblock.init | 1 + net/adblock/files/www/adblock/adblock.html | 6 - 8 files changed, 129 insertions(+), 223 deletions(-) delete mode 100644 net/adblock/files/www/adblock/adblock.html diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 84b775179..ca3a183ad 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=0.80.1 +PKG_VERSION:=0.90.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -25,7 +25,7 @@ endef define Package/$(PKG_NAME)/description Powerful adblock script to block ad/abuse domains. Currently the script supports 15 domain blacklist sites plus manual black- and whitelist overrides. -Please see README.md in /etc/adblock for further information. +Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. endef @@ -62,7 +62,6 @@ define Package/$(PKG_NAME)/install $(INSTALL_CONF) ./files/adblock.conf $(1)/etc/adblock/adblock.conf.default $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock/ $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock/ - $(INSTALL_CONF) ./files/README.md $(1)/etc/adblock/ $(INSTALL_DIR) $(1)/www/adblock $(INSTALL_DATA) ./files/www/adblock/* $(1)/www/adblock/ diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 88a58ef2b..838d5c0c8 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -46,15 +46,13 @@ When the dns server on your router receives dns requests, you will sort out quer * adblock source list parsing by fast & flexible regex rulesets * additional white- and blacklist support for manual overrides * quality checks during & after update of adblock lists to ensure a reliable dnsmasq service -* wan update check, to wait for an active wan uplink before update * basic adblock statistics via iptables packet counters for each chain * status & error logging to stdout and syslog * use of dynamic uhttpd instance as adblock pixel server +* use of dynamic iptables ruleset for adblock related redirects/rejects * openwrt init system support (start/stop/restart/reload) * hotplug support, adblock start will be triggered by wan 'ifup' event -* optional features (disabled by default): - * adblock list backup/restore - * debug logging to separate file +* optional: adblock list backup/restore (disabled by default) ## Prerequisites * [openwrt](https://openwrt.org), tested with latest stable release (Chaos Calmer 15.05) and with current trunk (Designated Driver > r47025) @@ -81,28 +79,29 @@ Thanks to Hannu Nyman for this great adblock LuCI frontend! ## Tweaks * there is no need to enable all blacklist sites at once, for normal use one to three adblock list sources should be sufficient -* if you really need to handle all blacklists at once add an usb stick or any other storage device to supersize your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details -* add static, personal domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), by default both lists are located in */etc/adblock* -* enable the backup/restore feature, to restore automatically the latest, stable backup of your adblock lists in case of any processing error -* enable the logging feature for continuous logfile writing to monitor the adblock runs over a longer period +* if you really need to handle all blacklists at once add an usb stick or any other storage device to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details +* add personal domain white- or blacklist entries as an additional blocklist source, one domain per line (wildcards & regex are not allowed!), by default both empty lists are located in */etc/adblock* +* enable the backup/restore feature, to restore automatically the latest stable backup of your adblock lists in case of any (partial) processing error (i.e. a single blocklist source server is down). Please use an (external) solid partition and *not* your volatile router temp directory for this * for a scheduled call of the adblock service via */etc/init.d/adblock start* add an appropriate crontab entry ## Further adblock config options * usually the adblock autodetection works quite well and no manual config overrides are needed, all options apply to 'global' adblock config section: - * adb\_enabled => main switch to enable/disable adblock service (default: '1' (enabled)) + * adb\_enabled => main switch to enable/disable adblock service (default: '1', enabled) * adb\_cfgver => config version string (do not change!) - adblock checks this entry and automatically applies the current config, if none or an older revision was found. * adb\_wanif => name of the logical wan interface (default: 'wan') * adb\_lanif => name of the logical lan interface (default: 'lan') * adb\_port => port of the adblock uhttpd instance (default: '65535') * adb\_nullipv4 => IPv4 blackhole ip address (default: '192.0.2.1') * adb\_nullipv6 => IPv6 blackhole ip address (default: '::ffff:c000:0201') + * adb\_forcedns => redirect all DNS queries to local dnsmasq resolver (default: '1', enabled) ## Background This adblock package is a dns/dnsmasq based adblock solution for openwrt. Queries to ad/abuse domains are never forwarded and always replied with a local IP address which may be IPv4 or IPv6. For that purpose adblock uses an ip address from the private 'TEST-NET-1' subnet (192.0.2.1 / ::ffff:c000:0201) by default. Furthermore all ad/abuse queries will be filtered by ip(6)tables and redirected to internal adblock pixel server (in PREROUTING chain) or rejected (in FORWARD or OUTPUT chain). -All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files. There is *no* adblock background daemon running, the (scheduled) start of the adblock service keeps only the adblock lists up-to-date. +All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files. +There is *no* adblock background daemon running, the (scheduled) start of the adblock service keeps only the adblock lists up-to-date. ## Support Please join the adblock discussion in this [openwrt forum thread](https://forum.openwrt.org/viewtopic.php?id=59803) or contact me by mail diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index faccc9640..d96ce4cca 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -1,8 +1,6 @@ #!/bin/sh -################################################# -# function library used by adblock-update.sh # -# written by Dirk Brenken (openwrt@brenken.org) # -################################################# +# function library used by adblock-update.sh +# written by Dirk Brenken (openwrt@brenken.org) ##################################### # f_envload: load adblock environment @@ -88,6 +86,7 @@ f_envparse() adb_nullipv6="::ffff:c000:0201" adb_blacklist="/etc/adblock/adblock.blacklist" adb_whitelist="/etc/adblock/adblock.whitelist" + adb_forcedns=1 # function to read global options by callback # @@ -110,7 +109,7 @@ f_envparse() # function to iterate through config list, read only options in "enabled" sections # - adb_cfglist="adb_backupdir adb_logfile adb_src" + adb_cfglist="adb_backupdir adb_src" unset adb_sources parse_config() { @@ -153,9 +152,8 @@ f_envparse() # set more script defaults (can't be overwritten by adblock config options) # - adb_cnt=0 - adb_minspace=20000 - adb_unique=1 + adb_count=0 + adb_minspace=12000 adb_tmpfile="$(mktemp -tu 2>/dev/null)" adb_tmpdir="$(mktemp -p /tmp -d 2>/dev/null)" adb_dnsdir="/tmp/dnsmasq.d" @@ -167,9 +165,7 @@ f_envparse() adb_fwdchain_ipv6="forwarding_rule" adb_outchain_ipv6="output_rule" adb_fetch="/usr/bin/wget" - unset adb_srclist - unset adb_revsrclist - unset adb_errsrclist + unset adb_srclist adb_revsrclist adb_errsrclist # set adblock source ruleset definitions # @@ -245,13 +241,13 @@ f_envcheck() check="$(printf "${pkg_list}" | grep "^ip6tables -" 2>/dev/null)" if [ -z "${check}" ] then - f_log "package 'ip6tables' not found, IPv6 support wíll be disabled" + f_log "package 'ip6tables' not found, IPv6 support will be disabled" unset adb_wanif6 else check="$(printf "${pkg_list}" | grep "^kmod-ipt-nat6 -" 2>/dev/null)" if [ -z "${check}" ] then - f_log "package 'kmod-ipt-nat6' not found, IPv6 support wíll be disabled" + f_log "package 'kmod-ipt-nat6' not found, IPv6 support will be disabled" unset adb_wanif6 fi fi @@ -270,14 +266,11 @@ f_envcheck() # if [ ! -r "${adb_blacklist}" ] then - rc=135 - f_log "adblock blacklist not found (${adb_blacklist})" "${rc}" - f_exit - elif [ ! -r "${adb_whitelist}" ] + f_log "adblock blacklist not found, source will be disabled" + fi + if [ ! -r "${adb_whitelist}" ] then - rc=135 - f_log "adblock whitelist not found (${adb_whitelist})" "${rc}" - f_exit + f_log "adblock whitelist not found, source will be disabled" fi # check adblock temp directory @@ -287,25 +280,30 @@ f_envcheck() f_space "${adb_tmpdir}" if [ "${space_ok}" = "false" ] then - rc=140 - f_log "not enough space in '${adb_tmpdir}', please supersize your temp directory" "${rc}" - f_exit + if [ $((av_space)) -le 2000 ] + then + rc=135 + f_log "not enough free space in '${adb_tmpdir}' (avail. ${av_space} kb)" "${rc}" + f_exit + else + f_log "not enough free space to handle all adblock list sources at once in '${adb_tmpdir}' (avail. ${av_space} kb)" + fi fi else - rc=140 + rc=135 f_log "temp directory not found" "${rc}" f_exit fi # memory check # - mem_total="$(awk '$1 ~ /^MemTotal/ {print $2}' "/proc/meminfo" 2>/dev/null)" - mem_free="$(awk '$1 ~ /^MemFree/ {print $2}' "/proc/meminfo" 2>/dev/null)" - mem_swap="$(awk '$1 ~ /^SwapTotal/ {print $2}' "/proc/meminfo" 2>/dev/null)" + mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)" + mem_free="$(awk '$1 ~ /^MemFree/ {printf $2}' "/proc/meminfo" 2>/dev/null)" + mem_swap="$(awk '$1 ~ /^SwapTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)" if [ $((mem_total)) -le 64000 ] && [ $((mem_swap)) -eq 0 ] then - adb_unique=0 - f_log "not enough memory, overall sort processing will be disabled (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})" + mem_ok="false" + f_log "not enough free memory, overall sort processing will be disabled (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})" fi # check backup configuration @@ -315,7 +313,7 @@ f_envcheck() f_space "${adb_backupdir}" if [ "${space_ok}" = "false" ] then - f_log "not enough space in '${adb_backupdir}', backup/restore will be disabled" + f_log "not enough free space in '${adb_backupdir}'(avail. ${av_space} kb), backup/restore will be disabled" backup_ok="false" else f_log "backup/restore will be enabled" @@ -326,34 +324,18 @@ f_envcheck() f_log "backup/restore will be disabled" fi - # check log configuration - # - adb_logdir="${adb_logfile%/*}" - if [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] - then - f_space "${adb_logdir}" - if [ "${space_ok}" = "false" ] - then - f_log "not enough space in '${adb_logdir}', logging will be disabled" - log_ok="false" - else - f_log "logging will be enabled" - log_ok="true" - fi - else - log_ok="false" - f_log "logging will be disabled" - fi - # check ipv4/iptables configuration # if [ -n "${adb_wanif4}" ] && [ -n "${adb_wandev4}" ] then f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-prerouting" "! -i ${adb_wandev4} -p tcp -d ${adb_nullipv4} -m multiport --dports 80,443 -j REDIRECT --to-ports ${adb_port}" - f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-dns" "! -i ${adb_wandev4} -p udp --dport 53 -j REDIRECT" - f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-dns" "! -i ${adb_wandev4} -p tcp --dport 53 -j REDIRECT" f_firewall "IPv4" "filter" "A" "${adb_fwdchain_ipv4}" "adb-forward" "! -i ${adb_wandev4} -d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable" f_firewall "IPv4" "filter" "A" "${adb_outchain_ipv4}" "adb-output" "! -i ${adb_wandev4} -d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable" + if [ $((adb_forcedns)) -eq 1 ] + then + f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-dns" "! -i ${adb_wandev4} -p udp --dport 53 -j REDIRECT" + f_firewall "IPv4" "nat" "A" "${adb_prechain_ipv4}" "adb-dns" "! -i ${adb_wandev4} -p tcp --dport 53 -j REDIRECT" + fi if [ "${fw_done}" = "true" ] then f_log "created volatile IPv4 firewall ruleset" @@ -366,10 +348,13 @@ f_envcheck() if [ -n "${adb_wanif6}" ] && [ -n "${adb_wandev6}" ] then f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-prerouting" "! -i ${adb_wandev6} -p tcp -d ${adb_nullipv6} -m multiport --dports 80,443 -j REDIRECT --to-ports ${adb_port}" - f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-dns" "! -i ${adb_wandev6} -p udp --dport 53 -j REDIRECT" - f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-dns" "! -i ${adb_wandev6} -p tcp --dport 53 -j REDIRECT" f_firewall "IPv6" "filter" "A" "${adb_fwdchain_ipv6}" "adb-forward" "! -i ${adb_wandev6} -d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable" f_firewall "IPv6" "filter" "A" "${adb_outchain_ipv6}" "adb-output" "! -i ${adb_wandev6} -d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable" + if [ $((adb_forcedns)) -eq 1 ] + then + f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-dns" "! -i ${adb_wandev6} -p udp --dport 53 -j REDIRECT" + f_firewall "IPv6" "nat" "A" "${adb_prechain_ipv6}" "adb-dns" "! -i ${adb_wandev6} -p tcp --dport 53 -j REDIRECT" + fi if [ "${fw_done}" = "true" ] then f_log "created volatile IPv6 firewall ruleset" @@ -384,37 +369,23 @@ f_envcheck() then if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] then - uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/adblock.html" -p "${adb_ipv4}:${adb_port}" -p "[${adb_ipv6}]:${adb_port}">/dev/null 2>&1 + uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_port}" -p "[${adb_ipv6}]:${adb_port}">/dev/null 2>&1 rc=${?} - if [ $((rc)) -eq 0 ] - then - f_log "created volatile uhttpd instance (${adb_ipv4}:${adb_port}, [${adb_ipv6}]:${adb_port})" - else - f_log "failed to initialize volatile uhttpd instance (${adb_ipv4}:${adb_port}, [${adb_ipv6}]:${adb_port})" "${rc}" - f_restore - fi elif [ -n "${adb_wanif4}" ] then - uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/adblock.html" -p "${adb_ipv4}:${adb_port}" >/dev/null 2>&1 + uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_port}" >/dev/null 2>&1 rc=${?} - if [ $((rc)) -eq 0 ] - then - f_log "created volatile uhttpd instance (${adb_ipv4}:${adb_port})" - else - f_log "failed to initialize volatile uhttpd instance (${adb_ipv4}:${adb_port})" "${rc}" - f_restore - fi elif [ -n "${adb_wanif6}" ] then - uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/adblock.html" -p "[${adb_ipv6}]:${adb_port}" >/dev/null 2>&1 + uhttpd -h "/www/adblock" -k 5 -N 200 -t 0 -T 1 -D -S -E "/index.html" -p "[${adb_ipv6}]:${adb_port}" >/dev/null 2>&1 rc=${?} - if [ $((rc)) -eq 0 ] - then - f_log "created volatile uhttpd instance ([${adb_ipv6}]:${adb_port})" - else - f_log "failed to initialize volatile uhttpd instance ([${adb_ipv6}]:${adb_port})" "${rc}" - f_restore - fi + fi + if [ $((rc)) -eq 0 ] + then + f_log "created volatile uhttpd instance" + else + f_log "failed to initialize volatile uhttpd instance" "${rc}" + f_restore fi fi @@ -447,7 +418,7 @@ f_depend() check="$(printf "${pkg_list}" | grep "^${package} -" 2>/dev/null)" if [ -z "${check}" ] then - rc=150 + rc=140 f_log "package '${package}' not found" "${rc}" f_exit fi @@ -494,8 +465,8 @@ f_firewall() fi } -################################################### -# f_log: log messages to stdout, syslog and logfile +########################################## +# f_log: log messages to stdout and syslog # f_log() { @@ -511,7 +482,7 @@ f_log() log_parm="-s" fi - # log to different output devices, set log class accordingly + # log to different output devices and set log class accordingly # if [ -n "${log_msg}" ] then @@ -522,10 +493,6 @@ f_log() log_msg="${log_msg}${log_rc}" fi /usr/bin/logger ${log_parm} -t "adblock[${adb_pid}] ${class}" "${log_msg}" - if [ "${log_ok}" = "true" ] - then - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") adblock[${adb_pid}] ${class}: ${log_msg}" >> "${adb_logfile}" - fi fi } @@ -540,7 +507,7 @@ f_space() # if [ -d "${mp}" ] then - av_space="$(df "${mp}" 2>/dev/null | tail -n1 2>/dev/null | awk '{print $4}')" + av_space="$(df "${mp}" 2>/dev/null | tail -n1 2>/dev/null | awk '{printf $4}')" if [ $((av_space)) -lt $((adb_minspace)) ] then space_ok="false" @@ -595,20 +562,19 @@ f_restore() if [ -n "${restore_done}" ] || [ -n "${rm_done}" ] then /etc/init.d/dnsmasq restart >/dev/null 2>&1 - sleep 2 + sleep 1 dns_status="$(ps 2>/dev/null | grep "[d]nsmasq" 2>/dev/null)" if [ -n "${dns_status}" ] then rc=0 + adb_count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)" if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] then - adb_count="$(($(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l) / 2))" - else - adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)" + adb_count="$((adb_count / 2))" fi f_log "adblock lists with overall ${adb_count} domains loaded" else - rc=160 + rc=145 f_log "dnsmasq restart failed, please check 'logread' output" "${rc}" fi fi @@ -621,12 +587,12 @@ f_restore() # f_exit() { - local ipv4_prerouting - local ipv4_forward - local ipv4_output - local ipv6_prerouting - local ipv6_forward - local ipv6_output + local ipv4_prerouting=0 + local ipv4_forward=0 + local ipv4_output=0 + local ipv6_prerouting=0 + local ipv6_forward=0 + local ipv6_output=0 local iptv4="/usr/sbin/iptables" local iptv6="/usr/sbin/ip6tables" @@ -647,34 +613,20 @@ f_exit() then if [ -n "${adb_wanif4}" ] then - ipv4_prerouting="$(${iptv4} -t nat -vnL | awk '$11 ~ /^adb-prerouting$/ {sum += $1} END {print sum}')" - ipv4_forward="$(${iptv4} -vnL | awk '$11 ~ /^adb-forward$/ {sum += $1} END {print sum}')" - ipv4_output="$(${iptv4} -vnL | awk '$11 ~ /^adb-output$/ {sum += $1} END {print sum}')" + ipv4_prerouting="$(${iptv4} -t nat -vnL | awk '$11 ~ /^adb-prerouting$/ {sum += $1} END {printf sum}')" + ipv4_forward="$(${iptv4} -vnL | awk '$11 ~ /^adb-forward$/ {sum += $1} END {printf sum}')" + ipv4_output="$(${iptv4} -vnL | awk '$11 ~ /^adb-output$/ {sum += $1} END {printf sum}')" fi if [ -n "${adb_wanif6}" ] then - ipv6_prerouting="$(${iptv6} -t nat -vnL | awk '$10 ~ /^adb-prerouting$/ {sum += $1} END {print sum}')" - ipv6_forward="$(${iptv6} -vnL | awk '$10 ~ /^adb-forward$/ {sum += $1} END {print sum}')" - ipv6_output="$(${iptv6} -vnL | awk '$10 ~ /^adb-output$/ {sum += $1} END {print sum}')" - fi - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] - then - f_log "adblock firewall statistics (IPv4/IPv6):" - f_log "${ipv4_prerouting}/${ipv6_prerouting} packets redirected in PREROUTING chain" - f_log "${ipv4_forward}/${ipv6_forward} packets rejected in FORWARD chain" - f_log "${ipv4_output}/${ipv6_output} packets rejected in OUTPUT chain" - elif [ -n "${adb_wanif4}" ] - then - f_log "adblock firewall statistics (IPv4):" - f_log "${ipv4_prerouting} packets redirected in PREROUTING chain" - f_log "${ipv4_forward} packets rejected in FORWARD chain" - f_log "${ipv4_output} packets rejected in OUTPUT chain" - elif [ -n "${adb_wanif6}" ] - then - f_log "${ipv6_prerouting} packets redirected in PREROUTING chain" - f_log "${ipv6_forward} packets rejected in FORWARD chain" - f_log "${ipv6_output} packets rejected in OUTPUT chain" + ipv6_prerouting="$(${iptv6} -t nat -vnL | awk '$10 ~ /^adb-prerouting$/ {sum += $1} END {printf sum}')" + ipv6_forward="$(${iptv6} -vnL | awk '$10 ~ /^adb-forward$/ {sum += $1} END {printf sum}')" + ipv6_output="$(${iptv6} -vnL | awk '$10 ~ /^adb-output$/ {sum += $1} END {printf sum}')" fi + f_log "adblock firewall statistics (IPv4/IPv6):" + f_log "${ipv4_prerouting}/${ipv6_prerouting} packets redirected in PREROUTING chain" + f_log "${ipv4_forward}/${ipv6_forward} packets rejected in FORWARD chain" + f_log "${ipv4_output}/${ipv6_output} packets rejected in OUTPUT chain" f_log "domain adblock processing finished successfully (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" elif [ $((rc)) -gt 0 ] then diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index 2fec96d74..32a48b7d3 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -1,30 +1,12 @@ #!/bin/sh -####################################################### -# ad/abuse domain blocking script for dnsmasq/openwrt # -# written by Dirk Brenken (openwrt@brenken.org) # -####################################################### - -# LICENSE -# ======== -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# +# ad/abuse domain blocking script for dnsmasq/openwrt +# written by Dirk Brenken (openwrt@brenken.org) + +# This is free software, licensed under the GNU General Public License v3. # You should have received a copy of the GNU General Public License # along with this program. If not, see . -############### -# environment # -############### - -# set the C locale, characters are single bytes, the charset is ASCII -# speeds up things like sort, grep etc. +# set the C locale # LC_ALL=C @@ -45,7 +27,7 @@ fi # get current directory, script- and openwrt version # adb_scriptdir="${0%/*}" -adb_scriptver="0.80.1" +adb_scriptver="0.90.0" openwrt_version="$(cat /etc/openwrt_version 2>/dev/null)" # source in adblock function library @@ -59,13 +41,9 @@ else exit ${rc} fi -################ -# main program # -################ - # call trap function on error signals (HUP, INT, QUIT, BUS, SEGV, TERM) # -trap "rc=253; f_log 'error signal received/trapped' '${rc}'; f_exit" 1 2 3 10 11 15 +trap "rc=250; f_log 'error signal received/trapped' '${rc}'; f_exit" 1 2 3 10 11 15 # load environment # @@ -93,15 +71,13 @@ then shalla_file="${adb_tmpdir}/shallalist.txt" src_name="shalla" adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" - list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)" - list_time="${list_time/*: /}" + list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}" 2>/dev/null)" f_log "=> (pre-)processing adblock source '${src_name}'" # only process shallalist archive with updated timestamp, # extract and merge only domains of selected shallalist categories # - shalla_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${adb_arc_shalla}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)" - shalla_time="${shalla_time/*: /}" + shalla_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${adb_arc_shalla}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}' 2>/dev/null)" if [ -z "${shalla_time}" ] then shalla_time="$(date)" @@ -162,8 +138,7 @@ do url="${src/\&ruleset=*/}" src_name="${src/*\&ruleset=rset_/}" adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" - list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)" - list_time="${list_time/*: /}" + list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}" 2>/dev/null)" f_log "=> processing adblock source '${src_name}'" # prepare find statement with active adblock list sources @@ -184,8 +159,7 @@ do then url_time="${shalla_time}" else - url_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${url}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)" - url_time="${url_time/*: /}" + url_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}' 2>/dev/null)" fi if [ -z "${url_time}" ] then @@ -262,19 +236,11 @@ do adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}" fi - # write preliminary adblock list footer + # write preliminary footer # if [ $((rc)) -eq 0 ] then - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] - then - count="$(($(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}") / 2))" - else - count="$(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}")" - fi - printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsfile}" - printf "%s\n" "# ${0##*/} (${adb_scriptver}) - ${count} ad/abuse domains blocked" >> "${adb_dnsfile}" - printf "%s\n" "# source: ${url}" >> "${adb_dnsfile}" + printf "%s\n" "#---------------------------------------------" >> "${adb_dnsfile}" printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}" f_log " domain merging finished" else @@ -356,44 +322,35 @@ then fi fi -# make separate adblock lists unique +# make separate adblock lists entries unique # -if [ $((adb_unique)) -eq 1 ] +if [ "${mem_ok}" != "false" ] then if [ -n "${adb_revsrclist}" ] then f_log "remove duplicates in separate adblock lists" - # generate a temporary, unique overall list + # generate a temporary unique overall list # - head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall" + head -qn -2 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall" # loop through all separate lists, ordered by size (ascending) # for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null) do - # check separate lists vs. overall list, - # rewrite only duplicate entries back to separate lists + # check original separate list vs. temporary overall list, + # rewrite only duplicate entries back to temporary separate list # list="${list/*./}" sort "${adb_dnsdir}/tmp.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" 2>/dev/null | uniq -d 2>/dev/null > "${adb_dnsdir}/tmp.${list}" - # remove these entries from overall list, - # rewrite only unique entries back to overall list + # rewrite only unique entries back to temporary overall list # tmp_unique="$(sort "${adb_dnsdir}/tmp.overall" "${adb_dnsdir}/tmp.${list}" 2>/dev/null | uniq -u 2>/dev/null)" printf "%s\n" "${tmp_unique}" > "${adb_dnsdir}/tmp.overall" - # write final adblocklist footer + # write unique result back to original separate list (with list footer) # - if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] - then - count="$(($(wc -l < "${adb_dnsdir}/tmp.${list}") / 2))" - else - count="$(wc -l < "${adb_dnsdir}/tmp.${list}")" - fi - printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsdir}/tmp.${list}" - printf "%s\n" "# ${0##*/} (${adb_scriptver}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/tmp.${list}" tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}" mv -f "${adb_dnsdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}" >/dev/null 2>&1 done @@ -401,14 +358,19 @@ then fi fi -# get overall count +# set separate list count & get overall count # -if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] -then - adb_count="$(($(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l) / 2))" -else - adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)" -fi +for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null) +do + list="${list/*./}" + count="$(head -qn -2 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)" + if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] + then + count=$((count / 2)) + fi + printf "%s\n" "# ${0##*/} (${adb_scriptver}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/${adb_dnsprefix}.${list}" + adb_count=$((adb_count + count)) +done # restart dnsmasq with newly generated or deleted adblock lists, # check dnsmasq startup afterwards @@ -416,13 +378,13 @@ fi if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ] || [ -n "${restore_done}" ] then /etc/init.d/dnsmasq restart >/dev/null 2>&1 - sleep 2 + sleep 1 dns_status="$(ps 2>/dev/null | grep "[d]nsmasq" 2>/dev/null)" if [ -n "${dns_status}" ] then f_log "adblock lists with overall ${adb_count} domains loaded" else - rc=105 + rc=100 f_log "dnsmasq restart failed, please check 'logread' output" "${rc}" f_restore fi diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 3c446d114..09d8cfa44 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -1,19 +1,16 @@ # adblock configuration, for further information -# see '/etc/adblock/README.md' +# see 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' config adblock 'global' option adb_enabled '1' - option adb_cfgver '0.80' + option adb_cfgver '0.90' option adb_blacklist '/etc/adblock/adblock.blacklist' option adb_whitelist '/etc/adblock/adblock.whitelist' + option adb_forcedns '1' config service 'backup' option enabled '0' - option adb_backupdir '/tmp' - -config service 'log' - option enabled '0' - option adb_logfile '/tmp/adb_debug.log' + option adb_backupdir '/mnt' config source 'adaway' option enabled '1' diff --git a/net/adblock/files/adblock.hotplug b/net/adblock/files/adblock.hotplug index 050701b1e..0aa619687 100644 --- a/net/adblock/files/adblock.hotplug +++ b/net/adblock/files/adblock.hotplug @@ -1,14 +1,16 @@ #!/bin/sh # -if [ -f "/var/run/adblock.pid" ] || [ "${ACTION}" != "ifup" ] +adb_pid="${$}" +adb_pidfile="/var/run/adblock.pid" +adb_logger="/usr/bin/logger" + +if [ -f "${adb_pidfile}" ] || [ "${ACTION}" != "ifup" ] then exit 0 fi . /lib/functions/network.sh -adb_pid="${$}" -adb_logger="/usr/bin/logger" network_find_wan adb_wanif4 network_find_wan6 adb_wanif6 diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index 2174fea60..5352d0e16 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -36,6 +36,7 @@ start() restart() { + stop start } diff --git a/net/adblock/files/www/adblock/adblock.html b/net/adblock/files/www/adblock/adblock.html deleted file mode 100644 index 29cdf586e..000000000 --- a/net/adblock/files/www/adblock/adblock.html +++ /dev/null @@ -1,6 +0,0 @@ - - - - - -