From 1c4a92c9f63d15656b55b6832ce584ee53e49625 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Fri, 20 Nov 2015 18:29:32 +0100 Subject: [PATCH] adblock: logging & documentation update - centralized logging via separate function to stdout, syslog and file - remove dependencies between helper functions - add two new options "adb_maxtime" and "adb_maxloop" - add description to every adblock config option (see adblock.conf.sample) - update README.md Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 7 +- net/adblock/files/README.md | 75 ++- net/adblock/files/adblock-helper.sh | 476 ++++++++++-------- net/adblock/files/adblock-update.sh | 70 +-- net/adblock/files/adblock.conf | 10 +- net/adblock/files/samples/adblock.conf.sample | 58 ++- .../files/samples/uhttpd.config.sample | 3 + 7 files changed, 417 insertions(+), 282 deletions(-) create mode 100644 net/adblock/files/samples/uhttpd.config.sample diff --git a/net/adblock/Makefile b/net/adblock/Makefile index f8c157b1b..45ec5d403 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=0.20.3 +PKG_VERSION:=0.21.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -24,8 +24,9 @@ endef define Package/$(PKG_NAME)/description powerful adblock script to block ad/abuse domains -When the dns server on your router receives dns requests, we will sort out queries that ask for the [A] resource records of ad servers -and return the local ip address of your router and the internal web server delivers a transparent pixel instead. +When the dns server on your router receives dns requests, we will sort out queries that ask for the [A] +resource records of ad servers and return the local ip address of your router and the internal web server +delivers a transparent pixel instead. The script supports the following domain blacklist sites: http://pgl.yoyo.org/adservers http://malwaredomains.com diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index f5f65abd1..0aa47904c 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -24,7 +24,7 @@ and return the local ip address of your router and the internal web server deliv * separate dynamic adblock network interface * separate dynamic uhttpd instance as pixel server * optional: quality checks and a powerful backup/restore handling to ensure a reliable dnsmasq service -* optional: adblock updates only on pre-defined wan interfaces +* optional: adblock updates only on pre-defined wan interfaces (useful for (mobile) multiwan setups) * optional: domain query logging as a background service to easily identify free and already blocked domains * optional: ntp time sync * optional: status & error logging (req. ntp time sync) @@ -36,20 +36,87 @@ and return the local ip address of your router and the internal web server deliv * wget (due to an openwrt bug still needed for certain https requests - see ticket #19621) * busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found) * optional: mounted usb stick or any other storage device to overcome limited memory resources on embedded router devices -* the above dependencies will be checked during package installation & script runtime, please check *logread -e "adblock"* for errors +* the above dependencies will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors ## Usage * select & install adblock package (*opkg install adblock*) -* configure /etc/config/adblock to your needs -* start /usr/bin/adblock-update.sh and check *logread -e "adblock"* for errors +* configure /etc/config/adblock to your needs, see additional comments in *adblock.conf.sample* +* by default openwrt uhttpd config is bind to 0.0.0.0 (to all ports of your router). For a working adblock configuration you have to bind uHTTPd to the standard LAN port only, please change *listen_http* and *listen_https* accordingly +* start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors ## Distributed samples * all sample configuration files stored in */etc/adblock/samples*. * to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*. +* to bind uhttpd to standard LAN port only, see *uhttpd.config.sample* * for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample*. * for scheduled call of *adblock-update.sh* see *root.crontab.sample*. * to redirect/force all dns queries to your router see *firwall.user.sample*. * for further dnsmasq tweaks see *dnsmasq.conf.sample*. +## Examples + + stdout excerpt for successful adblock run: + + root@pi2wrt:~# /usr/bin/adblock-update.sh + adblock[17771] info : domain adblock processing started (0.21.0) + adblock[17771] info : get wan/update interface (wlan1), after 0 loops + adblock[17771] info : get ntp time sync (0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org 3.pool.ntp.org), after 0 loops + adblock[17771] info : shallalist archive download finished + adblock[17771] info : shallalist archive extraction finished + adblock[17771] info : shallalist (pre-)processing finished (adv costtraps downloads spyware tracker warez) + adblock[17771] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2426 entries) + adblock[17771] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 15275 entries) + adblock[17771] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 3 entries) + adblock[17771] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries) + adblock[17771] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 11 entries) + adblock[17771] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries) + adblock[17771] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries) + adblock[17771] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13852 entries) + adblock[17771] info : source download finished (file:////tmp/tmp.emlDeH/shallalist.txt, 36961 entries) + adblock[17771] info : source download finished (file:///etc/adblock/adblock.blacklist, 1 entries) + adblock[17771] info : new adblock list with 73090 domains loaded, backup generated + adblock[17771] info : domain adblock processing finished (0.21.0) + + + generated domain blocklist for dnsmasq: + + address=/0-29.com/192.168.2.1 + address=/0-2u.com/192.168.2.1 + address=/0.r.msn.com/192.168.2.1 + address=/00.devoid.us/192.168.2.1 + address=/000007.ru/192.168.2.1 + [...] + address=/zzz.cn/192.168.2.1 + address=/zzzjsh.com/192.168.2.1 + #################################################### + # last adblock list update: 20.11.2015 - 18:00:02 + # adblock-update.sh (0.21.0) - 73087 ad/abuse domains blocked + # domain blacklist sources: + # http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext + # http://mirror1.malwaredomains.com/files/justdomains + # https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist + # https://feodotracker.abuse.ch/blocklist/?download=domainblocklist + # https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist + # http://www.dshield.org/feeds/suspiciousdomains_Low.txt + # http://spam404bl.com/spam404scamlist.txt + # http://winhelp2002.mvps.org/hosts.txt + # file:////tmp/tmp.CLBLNF/shallalist.txt + # file:///etc/adblock/adblock.blacklist + ##### + # /etc/adblock/adblock.whitelist + #################################################### + + + domain query log excerpt: + + query[A] www.seenby.de from fe80::6257:18ff:fe6b:4667 + query[A] tarifrechner.heise.de from 192.168.1.131 + query[A] www.mittelstandswiki.de from fe80::6257:18ff:fe6b:4667 + query[A] ad.doubleclick.net from 192.168.1.131 + ad.doubleclick.net is 192.168.2.1 + + +The first three queries are OK (not blocked), the last one has been blocked and answered by local dnsmasq instance. + Have fun! Dirk diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index 5b795ebb6..25f9c5ec2 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -15,9 +15,9 @@ f_envload() then . /lib/functions.sh else - /usr/bin/logger -t "adblock[${pid}]" "error: openwrt function library not found" + rc=510 + f_log "openwrt function library not found" "${rc}" f_deltemp - exit 10 fi # source in openwrt json helpers library @@ -26,9 +26,9 @@ f_envload() then . "/usr/share/libubox/jshn.sh" else - /usr/bin/logger -t "adblock[${pid}]" "error: openwrt json helpers library not found" + rc=515 + f_log "openwrt json helpers library not found" "${rc}" f_deltemp - exit 15 fi # get list with all installed openwrt packages @@ -36,9 +36,9 @@ f_envload() pkg_list="$(opkg list-installed 2>/dev/null)" if [ -z "${pkg_list}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: empty openwrt package list" + rc=520 + f_log "empty openwrt package list" "${rc}" f_deltemp - exit 20 fi } @@ -133,11 +133,8 @@ f_envparse() # set temp variables and counter # - adb_tmpfile="$(mktemp -tu)" - adb_tmpdir="$(mktemp -d)" - cnt=0 - max_cnt=30 - max_time=60 + adb_tmpfile="$(mktemp -tu 2>/dev/null)" + adb_tmpdir="$(mktemp -d 2>/dev/null)" # set adblock source ruleset definitions # @@ -160,13 +157,26 @@ f_envparse() # f_envcheck() { + # check required config variables + # + adb_varlist="adb_ip adb_dev adb_if adb_domain adb_minspace adb_maxloop adb_maxtime adb_blacklist adb_whitelist" + for var in ${adb_varlist} + do + if [ -z "$(eval printf \"\$"${var}"\")" ] + then + rc=525 + f_log "missing adblock config option (${var})" "${rc}" + f_deltemp + fi + done + # check adblock network device configuration # if [ ! -d "/sys/class/net/${adb_dev}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network device input (${adb_dev})" + rc=530 + f_log "invalid adblock network device input (${adb_dev})" "${rc}" f_deltemp - exit 25 fi # check adblock network interface configuration @@ -175,9 +185,9 @@ f_envcheck() banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')" if [ -n "${check_if}" ] || [ -n "${banned_if}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network interface input (${adb_if})" + rc=535 + f_log "invalid adblock network interface input (${adb_if})" "${rc}" f_deltemp - exit 30 fi # check adblock ip address configuration @@ -185,43 +195,56 @@ f_envcheck() check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')" if [ -z "${check_ip}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock ip address input (${adb_ip})" + rc=540 + f_log "invalid adblock ip address input (${adb_ip})" "${rc}" f_deltemp - exit 35 fi # check adblock blacklist/whitelist configuration # if [ ! -r "${adb_blacklist}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: adblock blacklist not found" + rc=545 + f_log "adblock blacklist not found" "${rc}" f_deltemp - exit 40 elif [ ! -r "${adb_whitelist}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: adblock whitelist not found" + rc=550 + f_log "adblock whitelist not found" "${rc}" f_deltemp - exit 45 fi - # check wan update configuration + # check adblock temp directory # - if [ -n "${adb_wandev}" ] + if [ -n "${adb_tmpdir}" ] && [ -d "${adb_tmpdir}" ] then - wan_ok="true" + f_space "${adb_tmpdir}" + tmp_ok="true" else - wan_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "info: wan update check will be disabled" + rc=555 + tmp_ok="false" + f_log "temp directory not found" "${rc}" + f_deltemp fi - # check ntp sync configuration + # check curl package dependency # - if [ -n "${adb_ntpsrv}" ] + check="$(printf "${pkg_list}" | grep "^curl")" + if [ -z "${check}" ] then - ntp_ok="true" - else - ntp_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "info: ntp time sync will be disabled" + rc=560 + f_log "curl package not found" "${rc}" + f_deltemp + fi + + # check wget package dependency + # + check="$(printf "${pkg_list}" | grep "^wget")" + if [ -z "${check}" ] + then + rc=565 + f_log "wget package not found" "${rc}" + f_deltemp fi # check backup configuration @@ -229,34 +252,17 @@ f_envcheck() adb_backupdir="${adb_backupfile%/*}" if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ] then + f_space "${adb_backupdir}" backup_ok="true" - adb_mounts="${adb_backupdir} ${adb_tmpdir}" else backup_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "info: backup/restore will be disabled" - fi - - # check error log configuration - # - adb_logdir="${adb_logfile%/*}" - if [ -n "${adb_logfile}" ] && [ "${adb_logfile}" = "/dev/stdout" ] - then - log_ok="true" - adb_logfile="/proc/self/fd/1" - elif [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] && [ "${ntp_ok}" = "true" ] - then - log_ok="true" - adb_mounts="${adb_mounts} ${adb_logdir}" - else - log_ok="false" - adb_logfile="/dev/null" - /usr/bin/logger -t "adblock[${pid}]" "info: error logging will be disabled" + f_log "backup/restore will be disabled" fi # check dns query log configuration # adb_querydir="${adb_queryfile%/*}" - query_pid="/var/run/adb_query.pid" + adb_querypid="/var/run/adb_query.pid" if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ] then # check find capabilities @@ -265,77 +271,59 @@ f_envcheck() if [ -z "${check}" ] then query_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "info: busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled" + f_log "busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled" else + f_space "${adb_querydir}" query_ok="true" query_name="${adb_queryfile##*/}" query_ip="${adb_ip//./\\.}" - adb_mounts="${adb_mounts} ${adb_querydir}" fi else query_ok="false" - if [ -s "${query_pid}" ] + f_log "dns query logging will be disabled" + if [ -s "${adb_querypid}" ] then - kill -9 $(cat "${query_pid}") 2>/dev/null - > "${query_pid}" - /usr/bin/logger -t "adblock[${pid}]" "info: remove old dns query log background process" + kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1 + f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}"))" + > "${adb_querypid}" fi - /usr/bin/logger -t "adblock[${pid}]" "info: dns query logging will be disabled" fi - # check mount points & space requirements + # check debug log configuration # - adb_mounts="${adb_mounts} ${adb_tmpdir}" - for mp in ${adb_mounts} - do - df "${mp}" 2>/dev/null |\ - tail -n1 |\ - while read filesystem overall used available scrap - do - av_space="${available}" - if [ $((av_space)) -eq 0 ] - then - /usr/bin/logger -t "adblock[${pid}]" "error: no space left on device/not mounted (${mp})" - exit 50 - elif [ $((av_space)) -lt $((adb_minspace)) ] - then - /usr/bin/logger -t "adblock[${pid}]" "error: not enough space left on device (${mp})" - exit 55 - fi - done - # subshell return code handling - # - rc=$? - if [ $((rc)) -ne 0 ] - then - f_deltemp - exit ${rc} - fi - done + adb_logdir="${adb_logfile%/*}" + if [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] + then + f_space "${adb_logdir}" + log_ok="true" + else + log_ok="false" + f_log "debug logging will be disabled" + fi - # check curl package dependency + # check wan update configuration # - check="$(printf "${pkg_list}" | grep "^curl")" - if [ -z "${check}" ] + if [ -n "${adb_wandev}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: curl package not found" - f_deltemp - exit 60 + f_wancheck "${adb_maxloop}" + else + wan_ok="false" + f_log "wan update check will be disabled" fi - # check wget package dependency + # check ntp sync configuration # - check="$(printf "${pkg_list}" | grep "^wget")" - if [ -z "${check}" ] + if [ -n "${adb_ntpsrv}" ] then - /usr/bin/logger -t "adblock[${pid}]" "error: wget package not found" - f_deltemp - exit 65 + f_ntpcheck "${adb_maxloop}" + else + ntp_ok="false" + f_log "ntp time sync will be disabled" fi # check dynamic/volatile adblock network interface configuration # - rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)" + rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf ${?})" if [ $((rc)) -ne 0 ] then json_init @@ -347,97 +335,148 @@ f_envcheck() json_close_array json_close_object ubus call network add_dynamic "$(json_dump)" - /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile network interface (${adb_if}, ${adb_ip})" + rc=${?} + if [ $((rc)) -eq 0 ] + then + f_log "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})" + else + f_log "failed to initialize new dynamic/volatile network interface (${adb_if}, ${adb_ip})" "${rc}" + f_deltemp + fi fi # check dynamic/volatile adblock uhttpd instance configuration # - rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf $?)" + rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf ${?})" if [ $((rc)) -ne 0 ] then - uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" - /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" + uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" >/dev/null 2>&1 + rc=${?} + if [ $((rc)) -eq 0 ] + then + f_log "created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" + else + f_log "failed to initialize new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" "${rc}" + f_deltemp + fi + fi +} + +################################################ +# f_log: log messages to stdout, syslog, logfile +# +f_log() +{ + local log_msg="${1}" + local log_rc="${2}" + local class="info " + if [ -n "${log_msg}" ] + then + if [ $((log_rc)) -ne 0 ] + then + class="error" + log_rc=", rc: ${log_rc}" + fi + /usr/bin/logger -s -t "adblock[${pid}] ${class}" "${log_msg}${log_rc}" + if [ "${log_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] + then + printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") adblock[${pid}] ${class}: ${log_msg}${log_rc}" >> "${adb_logfile}" + fi + fi +} + +################################################ +# f_space: check mount points/space requirements +# +f_space() +{ + local mp="${1}" + if [ -d "${mp}" ] + then + df "${mp}" 2>/dev/null |\ + tail -n1 |\ + while read filesystem overall used available scrap + do + av_space="${available}" + if [ $((av_space)) -eq 0 ] + then + rc=570 + f_log "no space left on device/not mounted (${mp})" "${rc}" + exit ${rc} + elif [ $((av_space)) -lt $((adb_minspace)) ] + then + rc=575 + f_log "not enough space left on device (${mp})" "${rc}" + exit ${rc} + fi + done + rc=${?} + if [ $((rc)) -eq 0 ] + then + space_ok="true" + else + space_ok="false" + f_deltemp + fi fi } -################################################### -# f_deltemp: delete temporary files and directories +#################################################### +# f_deltemp: delete temp files, directories and exit # f_deltemp() { if [ -f "${adb_tmpfile}" ] then - rm -f "${adb_tmpfile}" 2>/dev/null + rm -f "${adb_tmpfile}" >/dev/null 2>&1 fi if [ -d "${adb_tmpdir}" ] then - rm -rf "${adb_tmpdir}" 2>/dev/null + rm -rf "${adb_tmpdir}" >/dev/null 2>&1 fi + f_log "domain adblock processing finished (${adb_version})" + exit ${rc} } -################################################################ -# f_remove: remove temporary files, start and maintain query log +#################################################### +# f_remove: maintain and (re-)start domain query log # f_remove() { - # delete temporary files and directories - # - f_deltemp - - # remove existing domain query log background process, - # do housekeeping and start a new process on daily basis - # if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] then query_date="$(date "+%Y%m%d")" - if [ -s "${query_pid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ] + if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ] then - kill -9 $(cat "${query_pid}") 2>/dev/null - > "${query_pid}" - find "${adb_backupdir}" -maxdepth 1 -type f -mtime +${adb_queryhistory} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null - /usr/bin/logger -t "adblock[${pid}]" "info: remove old dns query log background process and do logfile housekeeping" + kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1 + find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null + f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping" + > "${adb_querypid}" fi - if [ ! -s "${query_pid}" ] + if [ ! -s "${adb_querypid}" ] then - ( logread -f 2>/dev/null & printf "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" & - /usr/bin/logger -t "adblock[${pid}]" "info: start new domain query log background process" + ( logread -f 2>/dev/null & printf ${!} > "${adb_querypid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" & + f_log "new domain query log background process started (pid: $(cat "${adb_querypid}"))" fi fi - - # final log entry - # - /usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing finished (${adb_version})" + f_deltemp } -##################################################### -# f_restore: if available, restore last adlist backup +################################################################ +# f_restore: restore last adblocklist backup and restart dnsmasq # f_restore() { - if [ -z "${restore_msg}" ] - then - restore_msg="unknown" - fi - if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ] then - cp -f "${adb_backupfile}" "${adb_dnsfile}" 2>/dev/null - /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, adlist backup restored" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${adb_logfile}" + cp -f "${adb_backupfile}" "${adb_dnsfile}" >/dev/null 2>&1 + f_log "adblocklist backup restored" else > "${adb_dnsfile}" - /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, empty adlist generated" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${adb_logfile}" + f_log="empty adblocklist generated" fi - - # restart dnsmasq - # /etc/init.d/dnsmasq restart >/dev/null 2>&1 - - # remove files and exit - # f_remove - exit 100 } ####################################################### @@ -445,35 +484,35 @@ f_restore() # f_wancheck() { - if [ "${wan_ok}" = "true" ] - then - # wait for wan update interface(s) - # - while [ $((cnt)) -le $((max_cnt)) ] + local cnt=0 + local cnt_max="${1}" + local dev + local dev_out + while [ $((cnt)) -le $((cnt_max)) ] + do + for dev in ${adb_wandev} do - for dev in ${adb_wandev} - do - if [ -d "/sys/class/net/${dev}" ] - then - dev_out="$(cat /sys/class/net/${dev}/operstate 2>/dev/null)" - if [ "${dev_out}" = "up" ] - then - /usr/bin/logger -t "adblock[${pid}]" "info: get wan/update interface: ${dev}, after ${cnt} loops" - break 2 - fi - fi - if [ $((cnt)) -eq $((max_cnt)) ] + if [ -d "/sys/class/net/${dev}" ] + then + dev_out="$(cat /sys/class/net/${dev}/operstate 2>/dev/null)" + rc=${?} + if [ "${dev_out}" = "up" ] then - wan_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "error: no wan/update interface(s) found (${adb_wandev})" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${adb_wandev})" >> "${adb_logfile}" - restore_msg="no wan/update interface(s)" - f_restore + wan_ok="true" + f_log "get wan/update interface (${dev}), after ${cnt} loops" + break 2 fi - done - sleep 1 - cnt=$((cnt + 1)) + fi done + sleep 1 + cnt=$((cnt + 1)) + done + if [ -z "${wan_ok}" ] + then + rc=580 + wan_ok="false" + f_log "no wan/update interface(s) found (${adb_wandev# })" "${rc}" + f_restore fi } @@ -482,72 +521,63 @@ f_wancheck() # f_ntpcheck() { - if [ "${ntp_ok}" = "true" ] + local cnt=0 + local cnt_max="${1}" + local ntp_pool + for srv in ${adb_ntpsrv} + do + ntp_pool="${ntp_pool} -p ${srv}" + done + while [ $((cnt)) -le $((cnt_max)) ] + do + /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1 + rc=${?} + if [ $((rc)) -eq 0 ] + then + ntp_ok="true" + f_log "get ntp time sync (${adb_ntpsrv# }), after ${cnt} loops" + break + fi + sleep 1 + cnt=$((cnt + 1)) + done + if [ -z "${ntp_ok}" ] then - # prepare ntp server pool - # - unset ntp_pool - for srv in ${adb_ntpsrv} - do - ntp_pool="${ntp_pool} -p ${srv}" - done - - # wait for ntp time sync - # - while [ $((cnt)) -le $((max_cnt)) ] - do - /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1 - rc=$? - if [ $((rc)) -eq 0 ] - then - /usr/bin/logger -t "adblock[${pid}]" "info: get ntp time sync (${adb_ntpsrv}), after ${cnt} loops" - break - fi - if [ $((cnt)) -eq $((max_cnt)) ] - then - ntp_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "error: ntp time sync failed (${adb_ntpsrv})" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${adb_ntpsrv})" >> "${adb_logfile}" - restore_msg="time sync failed" - f_restore - fi - sleep 1 - cnt=$((cnt + 1)) - done + rc=585 + ntp_ok="false" + f_log "ntp time sync failed (${adb_ntpsrv# })" "${rc}" + f_restore fi } -################################################################# -# f_dnscheck: dnsmasq health check with newly generated blocklist +#################################################################### +# f_dnscheck: dnsmasq health check with newly generated adblock list # f_dnscheck() { - # check 1: dnsmasq startup - # + local dns_status dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")" + rc=${?} if [ -z "${dns_status}" ] then - # check 2: nslookup probe - # dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")" + rc=${?} if [ -z "${dns_status}" ] then - # create backup of new block list only, if both checks are OK and backup enabled - # if [ "${backup_ok}" = "true" ] then - cp -f "${adb_dnsfile}" "${adb_backupfile}" 2>/dev/null - /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, backup generated" + cp -f "${adb_dnsfile}" "${adb_backupfile}" >/dev/null 2>&1 + f_log "new adblock list with ${adb_count} domains loaded, backup generated" else - /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, no backup" + f_log "new adblock list with ${adb_count} domains loaded, no backup" fi else - restore_msg="nslookup probe failed" + f_log "nslookup probe failed" "${rc}" f_restore fi else - restore_msg="dnsmasq probe failed" - f_restore + f_log "dnsmasq probe failed" "${rc}" + f_restore fi } @@ -556,9 +586,10 @@ f_dnscheck() # f_footer() { + local url adb_count="$(wc -l < "${adb_dnsfile}")" - printf "%s\n" "###################################################" >> "${adb_dnsfile}" - printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}" + printf "%s\n" "####################################################" >> "${adb_dnsfile}" + printf "%s\n" "# last adblock list update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}" printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}" printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}" for src in ${adb_sources} @@ -566,8 +597,7 @@ f_footer() url="${src//\&ruleset=*/}" printf "%s\n" "# ${url}" >> "${adb_dnsfile}" done - printf "%s\n" "###################################################" >> "${adb_dnsfile}" - printf "%s\n" "# domain whitelist source:" >> "${adb_dnsfile}" + printf "%s\n" "#####" >> "${adb_dnsfile}" printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}" - printf "%s\n" "###################################################" >> "${adb_dnsfile}" + printf "%s\n" "####################################################" >> "${adb_dnsfile}" } diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index 2fb49b645..86244816f 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -25,11 +25,11 @@ # set script version # -adb_version="0.20.3" +adb_version="0.21.0" # get current pid and script directory # -pid=$$ +pid=${$} adb_scriptdir="${0%/*}" # source in adblock function library @@ -38,8 +38,9 @@ if [ -r "${adb_scriptdir}/adblock-helper.sh" ] then . "${adb_scriptdir}/adblock-helper.sh" else - /usr/bin/logger -t "adblock[${pid}]" "error: adblock function library not found" - exit 200 + rc=500 + /usr/bin/logger -s -t "adblock[${pid}] error" "adblock function library not found, rc: ${rc}" + exit ${rc} fi ################ @@ -48,11 +49,11 @@ fi # call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM) # -trap "restore_msg='trap error'; f_restore" 1 2 3 10 11 15 +trap "f_log 'trap error' '600'; f_restore" 1 2 3 10 11 15 # start logging # -/usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing started (${adb_version})" +f_log "domain adblock processing started (${adb_version})" # load environment # @@ -66,14 +67,6 @@ f_envparse # f_envcheck -# check ntp time sync -# -f_ntpcheck - -# check wan update interface(s) -# -f_wancheck - # check/start shallalist (pre-)processing # if [ -n "${adb_arc_shalla}" ] @@ -82,29 +75,25 @@ then # shalla_archive="${adb_tmpdir}/shallalist.tar.gz" shalla_file="${adb_tmpdir}/shallalist.txt" - curl --insecure --max-time "${max_time}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null - rc=$? + curl --insecure --max-time "${adb_maxtime}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null + rc=${?} if [ $((rc)) -eq 0 ] then - /usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive download finished" + f_log "shallalist archive download finished" else - /usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive download failed (${adb_arc_shalla})" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${adb_arc_shalla})" >> "${adb_logfile}" - restore_msg="archive download failed" + f_log "shallalist archive download failed (${adb_arc_shalla})" "${rc}" f_restore fi # extract shallalist archive # tar -xzf "${shalla_archive}" -C "${adb_tmpdir}" 2>/dev/null - rc=$? + rc=${?} if [ $((rc)) -eq 0 ] then - /usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive extraction finished" + f_log "shallalist archive extraction finished" else - /usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive extraction failed" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${adb_logfile}" - restore_msg="archive extraction failed" + f_log "shallalist archive extraction failed" "${rc}" f_restore fi @@ -115,10 +104,10 @@ then do if [ -f "${adb_tmpdir}/BL/${category}/domains" ] then - cat "${adb_tmpdir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null - rc=$? + cat "${adb_tmpdir}/BL/${category}/domains" 2>/dev/null >> "${shalla_file}" + rc=${?} else - rc=220 + rc=505 fi if [ $((rc)) -ne 0 ] then @@ -131,11 +120,9 @@ then if [ $((rc)) -eq 0 ] then adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla" - /usr/bin/logger -t "adblock[${pid}]" "info: shallalist (pre-)processing finished (${adb_cat_shalla})" + f_log "shallalist (pre-)processing finished (${adb_cat_shalla# })" else - /usr/bin/logger -t "adblock[${pid}]" "error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})" >> "${adb_logfile}" - restore_msg="shallalist merge failed" + f_log "shallalist (pre-)processing failed (${adb_cat_shalla# })" "${rc}" f_restore fi fi @@ -152,11 +139,11 @@ do check_url="$(printf "${url}" | sed -n '/^https:/p')" if [ -n "${check_url}" ] then - tmp_var="$(wget --timeout="${max_time}" --tries=1 --output-document=- "${url}" 2>/dev/null)" - rc=$? + tmp_var="$(wget --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)" + rc=${?} else - tmp_var="$(curl --insecure --max-time "${max_time}" "${url}" 2>/dev/null)" - rc=$? + tmp_var="$(curl --insecure --max-time "${adb_maxtime}" "${url}" 2>/dev/null)" + rc=${?} fi # check download result and prepare domain output by regex patterns @@ -165,15 +152,13 @@ do then eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')" tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[A-Z]' '[a-z]')" - adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)" - /usr/bin/logger -t "adblock[${pid}]" "info: source download finished (${url}, ${adb_count} entries)" + count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)" + f_log "source download finished (${url}, ${count} entries)" elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ] then - /usr/bin/logger -t "adblock[${pid}]" "info: empty source download finished (${url})" + f_log "empty source download finished (${url})" else - /usr/bin/logger -t "adblock[${pid}]" "error: source download failed (${url})" - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${adb_logfile}" - restore_msg="download failed" + f_log "source download failed (${url})" "${rc}" f_restore fi done @@ -203,4 +188,3 @@ f_dnscheck # remove files and exit # f_remove -exit 0 diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 5fcacc0b6..7366051f5 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -5,7 +5,9 @@ config adblock "global" option adb_dev "eth0" option adb_if "adblock" option adb_domain "heise.de" - option adb_minspace "100000" + option adb_minspace "20000" + option adb_maxloop "5" + option adb_maxtime "60" option adb_blacklist "/etc/adblock/adblock.blacklist" option adb_whitelist "/etc/adblock/adblock.whitelist" @@ -24,13 +26,13 @@ config service "backup" option enabled "0" option adb_backupfile "/tmp/adlist.backup" -config service "errorlog" +config service "debuglog" option enabled "0" - option adb_logfile "/tmp/error.log" + option adb_logfile "/tmp/adb_debug.log" config service "querylog" option enabled "0" - option adb_queryfile "/tmp/query.log" + option adb_queryfile "/tmp/adb_query.log" option adb_queryhistory "1" config source "yoyo" diff --git a/net/adblock/files/samples/adblock.conf.sample b/net/adblock/files/samples/adblock.conf.sample index 5fcacc0b6..2b5034e32 100644 --- a/net/adblock/files/samples/adblock.conf.sample +++ b/net/adblock/files/samples/adblock.conf.sample @@ -1,18 +1,52 @@ -# adblock configuration +# adblock configuration sample +# + +# generic options (always required) # config adblock "global" + # ip address of the local adblock interface/uhttpd instance, + # needs to be a different subnet from the normal LAN option adb_ip "192.168.2.1" + + # name of the physical adblock network device (check /sys/class/net/), + # should point to the default lan interface option adb_dev "eth0" + + # name of the adblock network interface and uhttpd instance option adb_if "adblock" + + # name of an "always accessible" domain, + # this domain will be used for the final nslookup check option adb_domain "heise.de" - option adb_minspace "100000" + + # minimum required space for adlist, backups & logfiles (in kbyte) + # if you don't use all adblock sources, you can reduce this value accordingly + option adb_minspace "20000" + + # number of retries for wancheck and ntpcheck (see below) + option adb_maxloop "5" + + # download timeout for every adblock source (in seconds) + option adb_maxtime "60" + + # full path to static domain blacklist file (one domain per line) option adb_blacklist "/etc/adblock/adblock.blacklist" + + # full path to static domain whitelist file (one domain per line) option adb_whitelist "/etc/adblock/adblock.whitelist" +# list of devices that are allowed for adblock updates (check /sys/class/net/), +# if no one found the last adlist backup will be used, +# useful for (mobile) multiwan setups +# disabled by default config service "wancheck" option enabled "0" list adb_wanlist "wan" +# list of ntp time server pools, +# check that ntp has adjusted the system time on this device, +# will be used for logfile writing and logfile housekeeping +# disabled by default config service "ntpcheck" option enabled "0" list adb_ntplist "0.pool.ntp.org" @@ -20,19 +54,33 @@ config service "ntpcheck" list adb_ntplist "2.pool.ntp.org" list adb_ntplist "3.pool.ntp.org" +# full path to backup file for adlist backups +# disabled by default config service "backup" option enabled "0" option adb_backupfile "/tmp/adlist.backup" -config service "errorlog" +# full path to debug logfile +# by default adblock logs to syslog and stdout only +# disabled by default +config service "debuglog" option enabled "0" - option adb_logfile "/tmp/error.log" + option adb_logfile "/tmp/adb_debug.log" +# full path to domain query logfile +# a background task will trace every dns request to file, to easily identify free and already blocked domains, +# for this to work, you've to enable the dnsmasq option "logqueries" too. +# the "queryhistory" option deletes query logfiles older than n days (req. busybox find with mtime support) +# disabled by default config service "querylog" option enabled "0" - option adb_queryfile "/tmp/query.log" + option adb_queryfile "/tmp/adb_query.log" option adb_queryhistory "1" +# different adblock list sources +# please do not change the urls listed below, +# enable/disable sources as needed +# for shallalist you can also enable/disable different ad categories config source "yoyo" option enabled "1" option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo" diff --git a/net/adblock/files/samples/uhttpd.config.sample b/net/adblock/files/samples/uhttpd.config.sample new file mode 100644 index 000000000..2fc61ad40 --- /dev/null +++ b/net/adblock/files/samples/uhttpd.config.sample @@ -0,0 +1,3 @@ + config uhttpd 'main' + list listen_http '192.168.1.1:80' + list listen_https '192.168.1.1:443'