From 6d7571049187807b085ef39d3d920b28ee8942d3 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Tue, 24 Nov 2015 22:29:05 +0100 Subject: [PATCH] adblock: reduce memory consumption * rework shallalist processing: significantly reduce memory consumption during archive extraction and merging. * considerable reduce memory consumption during adblock source processing. * considerable reduce memory consumption of sort (sorts only the domain list and not the bigger dnsmasq file) other changes: * auto detection/defaults for adb_if, adb_dev, adb_ntpsrv, adb_maxloop, adb_maxtime and adb_minspace - these options can be safely removed from previous adblock configuration file * check total memory and main uhttpd configuration on startup * documentation update Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 6 +- net/adblock/files/README.md | 40 +++---- net/adblock/files/adblock-helper.sh | 100 ++++++++++++++---- net/adblock/files/adblock-update.sh | 76 +++++++------ net/adblock/files/adblock.conf | 9 -- net/adblock/files/samples/adblock.conf.sample | 28 +---- .../files/samples/uhttpd.config.sample | 3 +- 7 files changed, 148 insertions(+), 114 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 45ec5d403..dbc04c199 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=0.21.0 +PKG_VERSION:=0.22.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -23,7 +23,7 @@ define Package/$(PKG_NAME) endef define Package/$(PKG_NAME)/description -powerful adblock script to block ad/abuse domains +Powerful adblock script to block ad/abuse domains When the dns server on your router receives dns requests, we will sort out queries that ask for the [A] resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead. @@ -38,6 +38,8 @@ The script supports the following domain blacklist sites: http://www.spam404.com http://winhelp2002.mvps.org +Please read README.md in /etc/adblock for further information. + endef define Package/$(PKG_NAME)/conffiles diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 0aa47904c..e56e81871 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -10,15 +10,15 @@ and return the local ip address of your router and the internal web server deliv ## Main Features * support of the following domain blacklist sites (free for private usage, for commercial use please check their individual licenses): - * [pgl.yoyo.org](http://pgl.yoyo.org/adservers) - * [malwaredomains.com](http://malwaredomains.com) - * [zeustracker.abuse.ch](https://zeustracker.abuse.ch) - * [feodotracker.abuse.ch](https://feodotracker.abuse.ch) - * [palevotracker.abuse.ch](https://palevotracker.abuse.ch) - * [dshield.org](http://dshield.org) - * [shallalist.de](http://www.shallalist.de) (tested with the categories "adv" "costtraps" "downloads" "spyware" "tracker" "warez") - * [spam404.com](http://www.spam404.com) - * [winhelp2002.mvps.org](http://winhelp2002.mvps.org) + * [pgl.yoyo.org](http://pgl.yoyo.org/adservers), approx. 2.500 entries + * [malwaredomains.com](http://malwaredomains.com), approx. 16.000 entries + * [zeustracker.abuse.ch](https://zeustracker.abuse.ch), currently down + * [feodotracker.abuse.ch](https://feodotracker.abuse.ch), approx. 10 entries + * [palevotracker.abuse.ch](https://palevotracker.abuse.ch), approx. 10 entries + * [dshield.org](http://dshield.org), approx. 4.500 entries + * [shallalist.de](http://www.shallalist.de) (tested with the categories "adv" "costtraps" "downloads" "spyware" "tracker" "warez"), approx. 37.000 entries + * [spam404.com](http://www.spam404.com), approx. 5.000 entries + * [winhelp2002.mvps.org](http://winhelp2002.mvps.org), approx. 15.000 entries * blocklist parsing by fast & flexible regex rulesets * additional white- and blacklist support for manual overrides * separate dynamic adblock network interface @@ -34,24 +34,26 @@ and return the local ip address of your router and the internal web server deliv * additional software packages: * curl * wget (due to an openwrt bug still needed for certain https requests - see ticket #19621) - * busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found) -* optional: mounted usb stick or any other storage device to overcome limited memory resources on embedded router devices -* the above dependencies will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors + * optional: busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found) + * optional: coreutils-sort for reliable sort results, even on low memory systems +* recommended: add an usb stick or any other storage device to supersize your /tmp directory with a swap partition (see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab)) +* the above dependencies and requirements will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors ## Usage * select & install adblock package (*opkg install adblock*) * configure /etc/config/adblock to your needs, see additional comments in *adblock.conf.sample* -* by default openwrt uhttpd config is bind to 0.0.0.0 (to all ports of your router). For a working adblock configuration you have to bind uHTTPd to the standard LAN port only, please change *listen_http* and *listen_https* accordingly +* at least configure the ip address of the local adblock interface/uhttpd instance, needs to be a different subnet from the normal LAN +* by default openwrts main uhttpd instance is bind to all ports of your router. For a working adblock setup you have to bind uhttpd to the standard LAN port only, please change listen_http accordingly * start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors ## Distributed samples -* all sample configuration files stored in */etc/adblock/samples*. -* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*. +* all sample configuration files stored in */etc/adblock/samples* +* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample* * to bind uhttpd to standard LAN port only, see *uhttpd.config.sample* -* for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample*. -* for scheduled call of *adblock-update.sh* see *root.crontab.sample*. -* to redirect/force all dns queries to your router see *firwall.user.sample*. -* for further dnsmasq tweaks see *dnsmasq.conf.sample*. +* for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample* +* for scheduled call of *adblock-update.sh* see *root.crontab.sample* +* to redirect/force all dns queries to your router see *firwall.user.sample* +* for further dnsmasq tweaks see *dnsmasq.conf.sample* ## Examples diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index 25f9c5ec2..b033700d0 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -47,6 +47,29 @@ f_envload() # f_envparse() { + # set the C locale, characters are single bytes, the charset is ASCII + # speeds up sort, grep etc., guarantees unique domains + # + LC_ALL=C + + # set initial defaults (may be overwritten by adblock config options) + # + adb_if="adblock" + adb_minspace="20000" + adb_maxtime="60" + adb_maxloop="5" + + # adblock device name auto detection + # derived from first entry in openwrt lan ifname config + # + adb_dev="$(uci get network.lan.ifname 2>/dev/null)" + adb_dev="${adb_dev/ *}" + + # adblock ntp server name auto detection + # derived from ntp list found in openwrt ntp server config + # + adb_ntpsrv="$(uci get system.ntp.server 2>/dev/null)" + # function to read/set global options by callback, # prepare list items and build option list for all others # @@ -150,6 +173,14 @@ f_envparse() # adb_dnsfile="/tmp/dnsmasq.d/adlist.conf" adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'" + + # remove unused environment variables + # + env_list="$(set | grep -o "CONFIG_[A-Za-z_]*")" + for var in ${env_list} + do + unset "${var}" 2>/dev/null + done } ############################################# @@ -170,22 +201,33 @@ f_envcheck() fi done + # check main uhttpd configuration + # + check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -o "0.0.0.0")" + if [ -n "${check_uhttpd}" ] + then + rc=530 + lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)" + f_log "main uhttpd instance listens to all network interfaces, please bind uhttpd to LAN only (${lan_ip})" "${rc}" + f_deltemp + fi + # check adblock network device configuration # if [ ! -d "/sys/class/net/${adb_dev}" ] then - rc=530 + rc=535 f_log "invalid adblock network device input (${adb_dev})" "${rc}" f_deltemp fi # check adblock network interface configuration # - check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')" + check_if="$(printf "${adb_if}" | sed -n '/[^._0-9A-Za-z]/p')" banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')" if [ -n "${check_if}" ] || [ -n "${banned_if}" ] then - rc=535 + rc=540 f_log "invalid adblock network interface input (${adb_if})" "${rc}" f_deltemp fi @@ -195,7 +237,7 @@ f_envcheck() check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')" if [ -z "${check_ip}" ] then - rc=540 + rc=545 f_log "invalid adblock ip address input (${adb_ip})" "${rc}" f_deltemp fi @@ -204,12 +246,12 @@ f_envcheck() # if [ ! -r "${adb_blacklist}" ] then - rc=545 + rc=550 f_log "adblock blacklist not found" "${rc}" f_deltemp elif [ ! -r "${adb_whitelist}" ] then - rc=550 + rc=555 f_log "adblock whitelist not found" "${rc}" f_deltemp fi @@ -221,7 +263,7 @@ f_envcheck() f_space "${adb_tmpdir}" tmp_ok="true" else - rc=555 + rc=560 tmp_ok="false" f_log "temp directory not found" "${rc}" f_deltemp @@ -229,24 +271,46 @@ f_envcheck() # check curl package dependency # - check="$(printf "${pkg_list}" | grep "^curl")" + check="$(printf "${pkg_list}" | grep "^curl -")" if [ -z "${check}" ] then - rc=560 + rc=565 f_log "curl package not found" "${rc}" f_deltemp fi # check wget package dependency # - check="$(printf "${pkg_list}" | grep "^wget")" + check="$(printf "${pkg_list}" | grep "^wget -")" if [ -z "${check}" ] then - rc=565 + rc=570 f_log "wget package not found" "${rc}" f_deltemp fi + # check ca-certificates package and set wget/curl parms accordingly + # + check="$(printf "${pkg_list}" | grep "^ca-certificates -")" + if [ -z "${check}" ] + then + curl_parm="--insecure" + wget_parm="--no-check-certificate" + else + unset curl_parm + unset wget_parm + fi + + # check total and swap memory + # + mem_total="$(cat /proc/meminfo | grep "MemTotal" | grep -o "[0-9]*")" + mem_free="$(cat /proc/meminfo | grep "MemFree" | grep -o "[0-9]*")" + swap_total="$(cat /proc/meminfo | grep "SwapTotal" | grep -o "[0-9]*")" + if [ $((mem_total)) -le 64000 ] && [ $((swap_total)) -eq 0 ] + then + f_log "please consider to add an external swap device to supersize your /tmp directory (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})" + fi + # check backup configuration # adb_backupdir="${adb_backupfile%/*}" @@ -341,7 +405,7 @@ f_envcheck() f_log "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})" else f_log "failed to initialize new dynamic/volatile network interface (${adb_if}, ${adb_ip})" "${rc}" - f_deltemp + f_remove fi fi @@ -357,7 +421,7 @@ f_envcheck() f_log "created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" else f_log "failed to initialize new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" "${rc}" - f_deltemp + f_remove fi fi } @@ -400,12 +464,12 @@ f_space() av_space="${available}" if [ $((av_space)) -eq 0 ] then - rc=570 + rc=575 f_log "no space left on device/not mounted (${mp})" "${rc}" exit ${rc} elif [ $((av_space)) -lt $((adb_minspace)) ] then - rc=575 + rc=580 f_log "not enough space left on device (${mp})" "${rc}" exit ${rc} fi @@ -431,7 +495,7 @@ f_deltemp() rm -f "${adb_tmpfile}" >/dev/null 2>&1 fi if [ -d "${adb_tmpdir}" ] - then + then rm -rf "${adb_tmpdir}" >/dev/null 2>&1 fi f_log "domain adblock processing finished (${adb_version})" @@ -509,7 +573,7 @@ f_wancheck() done if [ -z "${wan_ok}" ] then - rc=580 + rc=585 wan_ok="false" f_log "no wan/update interface(s) found (${adb_wandev# })" "${rc}" f_restore @@ -543,7 +607,7 @@ f_ntpcheck() done if [ -z "${ntp_ok}" ] then - rc=585 + rc=590 ntp_ok="false" f_log "ntp time sync failed (${adb_ntpsrv# })" "${rc}" f_restore diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index 86244816f..4c198ac7a 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -25,7 +25,7 @@ # set script version # -adb_version="0.21.0" +adb_version="0.22.0" # get current pid and script directory # @@ -67,7 +67,7 @@ f_envparse # f_envcheck -# check/start shallalist (pre-)processing +# start shallalist (pre-)processing # if [ -n "${adb_arc_shalla}" ] then @@ -75,7 +75,7 @@ then # shalla_archive="${adb_tmpdir}/shallalist.tar.gz" shalla_file="${adb_tmpdir}/shallalist.txt" - curl --insecure --max-time "${adb_maxtime}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null + curl "${curl_parm}" --max-time "${adb_maxtime}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null rc=${?} if [ $((rc)) -eq 0 ] then @@ -85,46 +85,31 @@ then f_restore fi - # extract shallalist archive - # - tar -xzf "${shalla_archive}" -C "${adb_tmpdir}" 2>/dev/null - rc=${?} - if [ $((rc)) -eq 0 ] - then - f_log "shallalist archive extraction finished" - else - f_log "shallalist archive extraction failed" "${rc}" - f_restore - fi - - # merge selected shallalist categories + # extract and merge only domains of selected shallalist categories # > "${shalla_file}" for category in ${adb_cat_shalla} do - if [ -f "${adb_tmpdir}/BL/${category}/domains" ] + tar -C "${adb_tmpdir}" -xzf "${shalla_archive}" BL/${category}/domains 2>/dev/null + rc=${?} + if [ $((rc)) -eq 0 ] then - cat "${adb_tmpdir}/BL/${category}/domains" 2>/dev/null >> "${shalla_file}" - rc=${?} + if [ -r "${adb_tmpdir}/BL/${category}/domains" ] + then + cat "${adb_tmpdir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null + fi else - rc=505 - fi - if [ $((rc)) -ne 0 ] - then - break + f_log "shallalist archive extraction failed (${category})" "${rc}" + f_restore fi done # finish shallalist (pre-)processing # - if [ $((rc)) -eq 0 ] - then - adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla" - f_log "shallalist (pre-)processing finished (${adb_cat_shalla# })" - else - f_log "shallalist (pre-)processing failed (${adb_cat_shalla# })" "${rc}" - f_restore - fi + rm -f "${shalla_archive}" >/dev/null 2>&1 + rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1 + adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla" + f_log "shallalist (pre-)processing finished (${adb_cat_shalla# })" fi # loop through active adblock domain sources, @@ -139,10 +124,10 @@ do check_url="$(printf "${url}" | sed -n '/^https:/p')" if [ -n "${check_url}" ] then - tmp_var="$(wget --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)" + tmp_var="$(wget "${wget_parm}" --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)" rc=${?} else - tmp_var="$(curl --insecure --max-time "${adb_maxtime}" "${url}" 2>/dev/null)" + tmp_var="$(curl "${curl_parm}" --max-time "${adb_maxtime}" "${url}" 2>/dev/null)" rc=${?} fi @@ -154,6 +139,11 @@ do tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[A-Z]' '[a-z]')" count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)" f_log "source download finished (${url}, ${count} entries)" + if [ "${url}" = "file:///${shalla_file}" ] + then + rm -f "${shalla_file}" >/dev/null 2>&1 + fi + unset tmp_var elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ] then f_log "empty source download finished (${url})" @@ -163,14 +153,20 @@ do fi done -# create empty destination file +# remove whitelist domains, sort domains and make them unique +# and finally rewrite ad/abuse domain information to dnsmasq file # > "${adb_dnsfile}" - -# rewrite ad/abuse domain information to dns file, -# remove duplicates and whitelist entries -# -grep -vxf "${adb_whitelist}" < "${adb_tmpfile}" | eval "${adb_dnsformat}" | sort -u 2>/dev/null >> "${adb_dnsfile}" +grep -vxf "${adb_whitelist}" < "${adb_tmpfile}" 2>/dev/null | sort -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null >> "${adb_dnsfile}" 2>/dev/null +rc=${?} +if [ $((rc)) -eq 0 ] +then + unset adb_tmpfile + f_log "domain merging finished" +else + f_log "domain merging failed" "${rc}" + f_restore +fi # write dns file footer # diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 7366051f5..650ac00c7 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -2,12 +2,7 @@ # config adblock "global" option adb_ip "192.168.2.1" - option adb_dev "eth0" - option adb_if "adblock" option adb_domain "heise.de" - option adb_minspace "20000" - option adb_maxloop "5" - option adb_maxtime "60" option adb_blacklist "/etc/adblock/adblock.blacklist" option adb_whitelist "/etc/adblock/adblock.whitelist" @@ -17,10 +12,6 @@ config service "wancheck" config service "ntpcheck" option enabled "0" - list adb_ntplist "0.pool.ntp.org" - list adb_ntplist "1.pool.ntp.org" - list adb_ntplist "2.pool.ntp.org" - list adb_ntplist "3.pool.ntp.org" config service "backup" option enabled "0" diff --git a/net/adblock/files/samples/adblock.conf.sample b/net/adblock/files/samples/adblock.conf.sample index 2b5034e32..d5f2aef46 100644 --- a/net/adblock/files/samples/adblock.conf.sample +++ b/net/adblock/files/samples/adblock.conf.sample @@ -8,51 +8,29 @@ config adblock "global" # needs to be a different subnet from the normal LAN option adb_ip "192.168.2.1" - # name of the physical adblock network device (check /sys/class/net/), - # should point to the default lan interface - option adb_dev "eth0" - - # name of the adblock network interface and uhttpd instance - option adb_if "adblock" - # name of an "always accessible" domain, # this domain will be used for the final nslookup check option adb_domain "heise.de" - # minimum required space for adlist, backups & logfiles (in kbyte) - # if you don't use all adblock sources, you can reduce this value accordingly - option adb_minspace "20000" - - # number of retries for wancheck and ntpcheck (see below) - option adb_maxloop "5" - - # download timeout for every adblock source (in seconds) - option adb_maxtime "60" - # full path to static domain blacklist file (one domain per line) option adb_blacklist "/etc/adblock/adblock.blacklist" # full path to static domain whitelist file (one domain per line) option adb_whitelist "/etc/adblock/adblock.whitelist" -# list of devices that are allowed for adblock updates (check /sys/class/net/), -# if no one found the last adlist backup will be used, -# useful for (mobile) multiwan setups +# list of wan devices that are allowed for adblock updates (check /sys/class/net/), +# if no one found the last adlist backup will be restored, +# useful only for (mobile) multiwan setups # disabled by default config service "wancheck" option enabled "0" list adb_wanlist "wan" -# list of ntp time server pools, # check that ntp has adjusted the system time on this device, # will be used for logfile writing and logfile housekeeping # disabled by default config service "ntpcheck" option enabled "0" - list adb_ntplist "0.pool.ntp.org" - list adb_ntplist "1.pool.ntp.org" - list adb_ntplist "2.pool.ntp.org" - list adb_ntplist "3.pool.ntp.org" # full path to backup file for adlist backups # disabled by default diff --git a/net/adblock/files/samples/uhttpd.config.sample b/net/adblock/files/samples/uhttpd.config.sample index 2fc61ad40..b6dad6601 100644 --- a/net/adblock/files/samples/uhttpd.config.sample +++ b/net/adblock/files/samples/uhttpd.config.sample @@ -1,3 +1,4 @@ +# main uhttpd instance listens only to internal LAN +# config uhttpd 'main' list listen_http '192.168.1.1:80' - list listen_https '192.168.1.1:443'