From 45f005d856b43e94116a44e8afac3af02a4b1cd4 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Sun, 29 Nov 2015 16:44:24 +0100 Subject: [PATCH] adblock: maintenance update * remove "downloads" category from shallalist default configuration, due to false positives (i.e. debian & microsoft update sites) * remove example white- & blacklist entries * further optimize shallalist handling * fix domain merging with empty whitelist * various minor fixes & enhancements * documentation update Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 4 +-- net/adblock/files/README.md | 52 +++++++++++++++-------------- net/adblock/files/adblock-helper.sh | 33 +++++++++--------- net/adblock/files/adblock-update.sh | 30 ++++++++--------- net/adblock/files/adblock.blacklist | 2 +- net/adblock/files/adblock.conf | 1 - net/adblock/files/adblock.whitelist | 5 +-- 7 files changed, 63 insertions(+), 64 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 3acc0ce6a..31444a1ca 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=0.22.1 +PKG_VERSION:=0.22.2 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -17,7 +17,7 @@ include $(INCLUDE_DIR)/package.mk define Package/$(PKG_NAME) SECTION:=net CATEGORY:=Network - TITLE:=powerful adblock script to block ad/abuse domains + TITLE:=Powerful adblock script to block ad/abuse domains DEPENDS:=+curl +wget PKGARCH:=all endef diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index e56e81871..106f75639 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -12,22 +12,24 @@ and return the local ip address of your router and the internal web server deliv * support of the following domain blacklist sites (free for private usage, for commercial use please check their individual licenses): * [pgl.yoyo.org](http://pgl.yoyo.org/adservers), approx. 2.500 entries * [malwaredomains.com](http://malwaredomains.com), approx. 16.000 entries - * [zeustracker.abuse.ch](https://zeustracker.abuse.ch), currently down + * [zeustracker.abuse.ch](https://zeustracker.abuse.ch), approx. 420 entries * [feodotracker.abuse.ch](https://feodotracker.abuse.ch), approx. 10 entries * [palevotracker.abuse.ch](https://palevotracker.abuse.ch), approx. 10 entries * [dshield.org](http://dshield.org), approx. 4.500 entries - * [shallalist.de](http://www.shallalist.de) (tested with the categories "adv" "costtraps" "downloads" "spyware" "tracker" "warez"), approx. 37.000 entries + * [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default), approx. 32.000 entries + * a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html) * [spam404.com](http://www.spam404.com), approx. 5.000 entries * [winhelp2002.mvps.org](http://winhelp2002.mvps.org), approx. 15.000 entries * blocklist parsing by fast & flexible regex rulesets * additional white- and blacklist support for manual overrides * separate dynamic adblock network interface * separate dynamic uhttpd instance as pixel server -* optional: quality checks and a powerful backup/restore handling to ensure a reliable dnsmasq service +* adblock quality checks after list update to ensure a reliable dnsmasq service +* optional: powerful adblock list backup/restore handling * optional: adblock updates only on pre-defined wan interfaces (useful for (mobile) multiwan setups) -* optional: domain query logging as a background service to easily identify free and already blocked domains +* optional: domain query logging as a background service to easily identify free and already blocked domains (see example output below) +* optional: status & error logging to separate file (req. ntp time sync) * optional: ntp time sync -* optional: status & error logging (req. ntp time sync) ## Prerequisites * [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work @@ -41,8 +43,9 @@ and return the local ip address of your router and the internal web server deliv ## Usage * select & install adblock package (*opkg install adblock*) -* configure /etc/config/adblock to your needs, see additional comments in *adblock.conf.sample* +* configure */etc/config/adblock* to your needs, see additional comments in *adblock.conf.sample* * at least configure the ip address of the local adblock interface/uhttpd instance, needs to be a different subnet from the normal LAN +* optional: add additional domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), both list are located in */etc/adblock* * by default openwrts main uhttpd instance is bind to all ports of your router. For a working adblock setup you have to bind uhttpd to the standard LAN port only, please change listen_http accordingly * start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors @@ -59,25 +62,24 @@ and return the local ip address of your router and the internal web server deliv stdout excerpt for successful adblock run: - root@pi2wrt:~# /usr/bin/adblock-update.sh - adblock[17771] info : domain adblock processing started (0.21.0) - adblock[17771] info : get wan/update interface (wlan1), after 0 loops - adblock[17771] info : get ntp time sync (0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org 3.pool.ntp.org), after 0 loops - adblock[17771] info : shallalist archive download finished - adblock[17771] info : shallalist archive extraction finished - adblock[17771] info : shallalist (pre-)processing finished (adv costtraps downloads spyware tracker warez) - adblock[17771] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2426 entries) - adblock[17771] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 15275 entries) - adblock[17771] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 3 entries) - adblock[17771] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries) - adblock[17771] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 11 entries) - adblock[17771] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries) - adblock[17771] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries) - adblock[17771] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13852 entries) - adblock[17771] info : source download finished (file:////tmp/tmp.emlDeH/shallalist.txt, 36961 entries) - adblock[17771] info : source download finished (file:///etc/adblock/adblock.blacklist, 1 entries) - adblock[17771] info : new adblock list with 73090 domains loaded, backup generated - adblock[17771] info : domain adblock processing finished (0.21.0) + adblock[11541] info : domain adblock processing started (0.22.2, r47665, 29.11.2015 14:58:11) + adblock[11541] info : wan update check will be disabled + adblock[11541] info : get ntp time sync (192.168.254.254), after 0 loops + adblock[11541] info : shallalist (pre-)processing started ... + adblock[11541] info : shallalist (pre-)processing finished (adv costtraps spyware tracker warez) + adblock[11541] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2423 entries) + adblock[11541] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 16016 entries) + adblock[11541] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 419 entries) + adblock[11541] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries) + adblock[11541] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 12 entries) + adblock[11541] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries) + adblock[11541] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries) + adblock[11541] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13635 entries) + adblock[11541] info : source download finished (file:////tmp/tmp.CgbMmO/shallalist.txt, 32446 entries) + adblock[11541] info : empty source download finished (file:///etc/adblock/adblock.blacklist) + adblock[11541] info : domain merging finished + adblock[11541] info : new adblock list with 69646 domains loaded, backup generated + adblock[11541] info : domain adblock processing finished (0.22.2, r47665, 29.11.2015 14:59:23) generated domain blocklist for dnsmasq: diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index 8f46a5b37..80ed1f646 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -195,7 +195,7 @@ f_envcheck() # check main uhttpd configuration # - check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -o "0.0.0.0")" + check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -Fo "0.0.0.0")" if [ -n "${check_uhttpd}" ] then rc=530 @@ -287,20 +287,20 @@ f_envcheck() if [ -z "${check}" ] then curl_parm="-q --insecure" - wget_parm="--no-config --no-check-certificate" + wget_parm="--no-config --no-hsts --no-check-certificate" else curl_parm="-q" - wget_parm="--no-config" + wget_parm="--no-config --no-hsts" fi # check total and swap memory # - mem_total="$(cat /proc/meminfo | grep "MemTotal" | grep -o "[0-9]*")" - mem_free="$(cat /proc/meminfo | grep "MemFree" | grep -o "[0-9]*")" - swap_total="$(cat /proc/meminfo | grep "SwapTotal" | grep -o "[0-9]*")" + mem_total="$(cat /proc/meminfo | grep -F "MemTotal" | grep -o "[0-9]*")" + mem_free="$(cat /proc/meminfo | grep -F "MemFree" | grep -o "[0-9]*")" + swap_total="$(cat /proc/meminfo | grep -F "SwapTotal" | grep -o "[0-9]*")" if [ $((mem_total)) -le 64000 ] && [ $((swap_total)) -eq 0 ] then - f_log "please consider to add an external swap device to supersize your /tmp directory (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})" + f_log "please consider adding an external swap device to supersize your /tmp directory (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})" fi # check backup configuration @@ -323,7 +323,7 @@ f_envcheck() then # check find capabilities # - check="$(find --help 2>&1 | grep "mtime")" + check="$(find --help 2>&1 | grep -F "mtime")" if [ -z "${check}" ] then query_ok="false" @@ -340,7 +340,7 @@ f_envcheck() if [ -s "${adb_querypid}" ] then kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1 - f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}"))" + f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}" 2>/dev/null))" > "${adb_querypid}" fi fi @@ -442,11 +442,12 @@ f_log() then class="error" log_rc=", rc: ${log_rc}" + log_msg="${log_msg}${log_rc}" fi - /usr/bin/logger -s -t "adblock[${pid}] ${class}" "${log_msg}${log_rc}" + /usr/bin/logger -s -t "adblock[${pid}] ${class}" "${log_msg}" if [ "${log_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] then - printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") adblock[${pid}] ${class}: ${log_msg}${log_rc}" >> "${adb_logfile}" + printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") adblock[${pid}] ${class}: ${log_msg}" >> "${adb_logfile}" fi fi } @@ -500,7 +501,7 @@ f_deltemp() then rm -rf "${adb_tmpdir}" >/dev/null 2>&1 fi - f_log "domain adblock processing finished (${adb_version}, ${openwrt_version})" + f_log "domain adblock processing finished (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" exit ${rc} } @@ -516,13 +517,13 @@ f_remove() then kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1 find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null - f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping" + f_log "remove old domain query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping" > "${adb_querypid}" fi if [ ! -s "${adb_querypid}" ] then - ( logread -f 2>/dev/null & printf ${!} > "${adb_querypid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" & - f_log "new domain query log background process started (pid: $(cat "${adb_querypid}"))" + (logread -f 2>/dev/null & printf ${!} > "${adb_querypid}") | grep -Eo "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" & + f_log "new domain query log background process started" fi fi f_deltemp @@ -626,7 +627,7 @@ f_dnscheck() rc=${?} if [ -z "${dns_status}" ] then - dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")" + dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep -F "${adb_ip}")" rc=${?} if [ -z "${dns_status}" ] then diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index 9bd421b9c..c5a074c2d 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -25,7 +25,7 @@ # set script version # -adb_version="0.22.1" +adb_version="0.22.2" # get current pid, script directory and openwrt version # @@ -54,7 +54,7 @@ trap "f_log 'trap error' '600'; f_restore" 1 2 3 10 11 15 # start logging # -f_log "domain adblock processing started (${adb_version}, ${openwrt_version})" +f_log "domain adblock processing started (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))" # load environment # @@ -74,14 +74,13 @@ if [ -n "${adb_arc_shalla}" ] then # download shallalist archive # + f_log "shallalist (pre-)processing started ..." shalla_archive="${adb_tmpdir}/shallalist.tar.gz" shalla_file="${adb_tmpdir}/shallalist.txt" curl ${curl_parm} --max-time "${adb_maxtime}" "${adb_arc_shalla}" --output "${shalla_archive}" 2>/dev/null rc=${?} - if [ $((rc)) -eq 0 ] + if [ $((rc)) -ne 0 ] then - f_log "shallalist archive download finished" - else f_log "shallalist archive download failed (${adb_arc_shalla})" "${rc}" f_restore fi @@ -91,15 +90,10 @@ then > "${shalla_file}" for category in ${adb_cat_shalla} do - tar -C "${adb_tmpdir}" -xzf "${shalla_archive}" BL/${category}/domains 2>/dev/null + tar -xOzf "${shalla_archive}" BL/${category}/domains 2>/dev/null >> "${shalla_file}" rc=${?} - if [ $((rc)) -eq 0 ] + if [ $((rc)) -ne 0 ] then - if [ -r "${adb_tmpdir}/BL/${category}/domains" ] - then - cat "${adb_tmpdir}/BL/${category}/domains" 2>/dev/null >> "${shalla_file}" - fi - else f_log "shallalist archive extraction failed (${category})" "${rc}" f_restore fi @@ -157,9 +151,15 @@ done # remove whitelist domains, sort domains and make them unique # and finally rewrite ad/abuse domain information to dnsmasq file # -> "${adb_dnsfile}" -grep -vxf "${adb_whitelist}" < "${adb_tmpfile}" 2>/dev/null | sort -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null >> "${adb_dnsfile}" -rc=${?} +if [ -s "${adb_whitelist}" ] +then + grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}" + rc=${?} +else + sort -u "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}" + rc=${?} +fi + if [ $((rc)) -eq 0 ] then rm -f "${adb_tmpfile}" >/dev/null 2>&1 diff --git a/net/adblock/files/adblock.blacklist b/net/adblock/files/adblock.blacklist index 3df75830c..8b1378917 100644 --- a/net/adblock/files/adblock.blacklist +++ b/net/adblock/files/adblock.blacklist @@ -1 +1 @@ -bild.de + diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 650ac00c7..ef7141668 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -63,7 +63,6 @@ config source "shalla" option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz" list adb_catlist "adv" list adb_catlist "costtraps" - list adb_catlist "downloads" list adb_catlist "spyware" list adb_catlist "tracker" list adb_catlist "warez" diff --git a/net/adblock/files/adblock.whitelist b/net/adblock/files/adblock.whitelist index 42e0a57c3..8b1378917 100644 --- a/net/adblock/files/adblock.whitelist +++ b/net/adblock/files/adblock.whitelist @@ -1,4 +1 @@ -downloads.openwrt.org -ftp.de.debian.org -download.eclipse.org -dl.sourceforge.net +